request-log-analyzer 1.0.2 → 1.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. data/.gitignore +10 -0
  2. data/DESIGN.rdoc +41 -0
  3. data/LICENSE +4 -4
  4. data/README.rdoc +38 -0
  5. data/Rakefile +6 -3
  6. data/bin/request-log-analyzer +70 -72
  7. data/lib/cli/command_line_arguments.rb +53 -53
  8. data/lib/cli/database_console.rb +26 -0
  9. data/lib/cli/database_console_init.rb +43 -0
  10. data/lib/cli/progressbar.rb +166 -189
  11. data/lib/cli/tools.rb +49 -0
  12. data/lib/request_log_analyzer/aggregator/database_inserter.rb +83 -0
  13. data/lib/request_log_analyzer/aggregator/echo.rb +17 -12
  14. data/lib/request_log_analyzer/aggregator/summarizer.rb +101 -63
  15. data/lib/request_log_analyzer/{aggregator/base.rb → aggregator.rb} +17 -13
  16. data/lib/request_log_analyzer/controller.rb +251 -98
  17. data/lib/request_log_analyzer/database/base.rb +114 -0
  18. data/lib/request_log_analyzer/database/connection.rb +38 -0
  19. data/lib/request_log_analyzer/database/request.rb +22 -0
  20. data/lib/request_log_analyzer/database/source.rb +13 -0
  21. data/lib/request_log_analyzer/database/warning.rb +14 -0
  22. data/lib/request_log_analyzer/database.rb +102 -0
  23. data/lib/request_log_analyzer/file_format/amazon_s3.rb +74 -0
  24. data/lib/request_log_analyzer/file_format/apache.rb +147 -0
  25. data/lib/request_log_analyzer/file_format/delayed_job.rb +55 -0
  26. data/lib/request_log_analyzer/file_format/merb.rb +65 -29
  27. data/lib/request_log_analyzer/file_format/mysql.rb +101 -0
  28. data/lib/request_log_analyzer/file_format/postgresql.rb +68 -0
  29. data/lib/request_log_analyzer/file_format/rack.rb +9 -0
  30. data/lib/request_log_analyzer/file_format/rails.rb +164 -78
  31. data/lib/request_log_analyzer/file_format/rails3.rb +86 -0
  32. data/lib/request_log_analyzer/file_format/rails_development.rb +12 -0
  33. data/lib/request_log_analyzer/file_format.rb +252 -58
  34. data/lib/request_log_analyzer/filter/anonymize.rb +39 -0
  35. data/lib/request_log_analyzer/filter/field.rb +19 -13
  36. data/lib/request_log_analyzer/filter/timespan.rb +25 -12
  37. data/lib/request_log_analyzer/filter.rb +30 -0
  38. data/lib/request_log_analyzer/line_definition.rb +69 -96
  39. data/lib/request_log_analyzer/log_processor.rb +31 -53
  40. data/lib/request_log_analyzer/mailer.rb +65 -0
  41. data/lib/request_log_analyzer/output/fancy_html.rb +49 -0
  42. data/lib/request_log_analyzer/output/fixed_width.rb +220 -0
  43. data/lib/request_log_analyzer/output/html.rb +187 -0
  44. data/lib/request_log_analyzer/output.rb +117 -0
  45. data/lib/request_log_analyzer/request.rb +125 -40
  46. data/lib/request_log_analyzer/source/database_loader.rb +87 -0
  47. data/lib/request_log_analyzer/source/log_parser.rb +297 -0
  48. data/lib/request_log_analyzer/source.rb +72 -0
  49. data/lib/request_log_analyzer/tracker/duration.rb +28 -64
  50. data/lib/request_log_analyzer/tracker/frequency.rb +108 -0
  51. data/lib/request_log_analyzer/tracker/hourly_spread.rb +76 -49
  52. data/lib/request_log_analyzer/tracker/numeric_value.rb +223 -0
  53. data/lib/request_log_analyzer/tracker/timespan.rb +54 -27
  54. data/lib/request_log_analyzer/tracker/traffic.rb +40 -0
  55. data/lib/request_log_analyzer/tracker.rb +101 -0
  56. data/lib/request_log_analyzer.rb +43 -13
  57. data/request-log-analyzer.gemspec +41 -0
  58. data/spec/database.yml +23 -0
  59. data/spec/fixtures/apache_combined.log +5 -0
  60. data/spec/fixtures/apache_common.log +10 -0
  61. data/spec/fixtures/decompression.log +12 -0
  62. data/spec/fixtures/decompression.log.bz2 +0 -0
  63. data/spec/fixtures/decompression.log.gz +0 -0
  64. data/spec/fixtures/decompression.log.zip +0 -0
  65. data/spec/fixtures/decompression.tar.gz +0 -0
  66. data/spec/fixtures/decompression.tgz +0 -0
  67. data/spec/fixtures/header_and_footer.log +6 -0
  68. data/spec/fixtures/merb_prefixed.log +9 -0
  69. data/spec/fixtures/mysql_slow_query.log +110 -0
  70. data/spec/fixtures/postgresql.log +2980 -0
  71. data/spec/fixtures/rails.db +0 -0
  72. data/spec/fixtures/rails_22.log +1 -1
  73. data/spec/fixtures/sinatra.log +99 -0
  74. data/spec/integration/command_line_usage_spec.rb +84 -0
  75. data/spec/integration/mailer_spec.rb +179 -0
  76. data/spec/integration/munin_plugins_rails_spec.rb +58 -0
  77. data/spec/integration/scout_spec.rb +152 -0
  78. data/spec/lib/helpers.rb +72 -0
  79. data/spec/lib/macros.rb +18 -0
  80. data/spec/lib/matchers.rb +77 -0
  81. data/spec/lib/mocks.rb +77 -0
  82. data/spec/lib/testing_format.rb +46 -0
  83. data/spec/spec_helper.rb +16 -59
  84. data/spec/unit/aggregator/database_inserter_spec.rb +93 -0
  85. data/spec/unit/aggregator/summarizer_spec.rb +26 -0
  86. data/spec/unit/controller/controller_spec.rb +41 -0
  87. data/spec/unit/controller/log_processor_spec.rb +18 -0
  88. data/spec/unit/database/base_class_spec.rb +183 -0
  89. data/spec/unit/database/connection_spec.rb +34 -0
  90. data/spec/unit/database/database_spec.rb +133 -0
  91. data/spec/unit/file_format/amazon_s3_format_spec.rb +67 -0
  92. data/spec/unit/file_format/apache_format_spec.rb +203 -0
  93. data/spec/unit/file_format/common_regular_expressions_spec.rb +53 -0
  94. data/spec/unit/file_format/delayed_job_format_spec.rb +83 -0
  95. data/spec/unit/file_format/file_format_api_spec.rb +69 -0
  96. data/spec/unit/file_format/format_autodetection_spec.rb +40 -0
  97. data/spec/unit/file_format/line_definition_spec.rb +75 -0
  98. data/spec/unit/file_format/merb_format_spec.rb +52 -0
  99. data/spec/unit/file_format/mysql_format_spec.rb +154 -0
  100. data/spec/unit/file_format/postgresql_format_spec.rb +65 -0
  101. data/spec/unit/file_format/rack_format_spec.rb +50 -0
  102. data/spec/unit/file_format/rails3_format_spec.rb +120 -0
  103. data/spec/unit/file_format/rails_format_spec.rb +180 -0
  104. data/spec/unit/filter/anonymize_filter_spec.rb +21 -0
  105. data/spec/unit/filter/field_filter_spec.rb +66 -0
  106. data/spec/unit/filter/filter_spec.rb +17 -0
  107. data/spec/unit/filter/timespan_filter_spec.rb +58 -0
  108. data/spec/unit/mailer_spec.rb +42 -0
  109. data/spec/unit/request_spec.rb +111 -0
  110. data/spec/unit/source/log_parser_spec.rb +119 -0
  111. data/spec/unit/tracker/duration_tracker_spec.rb +49 -0
  112. data/spec/unit/tracker/frequency_tracker_spec.rb +88 -0
  113. data/spec/unit/tracker/hourly_spread_spec.rb +79 -0
  114. data/spec/unit/tracker/numeric_value_tracker_spec.rb +166 -0
  115. data/spec/unit/tracker/timespan_tracker_spec.rb +73 -0
  116. data/spec/unit/tracker/tracker_api_spec.rb +125 -0
  117. data/spec/unit/tracker/traffic_tracker_spec.rb +28 -0
  118. data/tasks/github-gem.rake +290 -139
  119. data/tasks/request_log_analyzer.rake +23 -7
  120. metadata +221 -94
  121. data/DESIGN +0 -14
  122. data/HACKING +0 -7
  123. data/README.textile +0 -36
  124. data/lib/cli/bashcolorizer.rb +0 -60
  125. data/lib/request_log_analyzer/aggregator/database.rb +0 -148
  126. data/lib/request_log_analyzer/filter/base.rb +0 -29
  127. data/lib/request_log_analyzer/log_parser.rb +0 -173
  128. data/lib/request_log_analyzer/source/base.rb +0 -42
  129. data/lib/request_log_analyzer/source/log_file.rb +0 -170
  130. data/lib/request_log_analyzer/tracker/base.rb +0 -54
  131. data/lib/request_log_analyzer/tracker/category.rb +0 -71
  132. data/spec/controller_spec.rb +0 -40
  133. data/spec/database_inserter_spec.rb +0 -101
  134. data/spec/file_format_spec.rb +0 -78
  135. data/spec/file_formats/spec_format.rb +0 -26
  136. data/spec/filter_spec.rb +0 -137
  137. data/spec/line_definition_spec.rb +0 -124
  138. data/spec/log_parser_spec.rb +0 -68
  139. data/spec/log_processor_spec.rb +0 -57
  140. data/spec/merb_format_spec.rb +0 -38
  141. data/spec/rails_format_spec.rb +0 -76
  142. data/spec/request_spec.rb +0 -72
  143. data/spec/summarizer_spec.rb +0 -9
  144. data/tasks/rspec.rake +0 -6
@@ -1,113 +1,151 @@
1
- require File.dirname(__FILE__) + '/../tracker/base'
2
-
3
1
  module RequestLogAnalyzer::Aggregator
4
2
 
5
3
  class Summarizer < Base
6
-
4
+
7
5
  class Definer
8
-
6
+
9
7
  attr_reader :trackers
10
-
8
+
9
+ # Initialize tracker array
11
10
  def initialize
12
11
  @trackers = []
13
12
  end
14
-
15
- def method_missing(tracker_method, *args)
16
- track(tracker_method, args.first)
13
+
14
+ # Initialize tracker summarizer by duping the trackers of another summarizer
15
+ # <tt>other</tt> The other Summarizer
16
+ def initialize_copy(other)
17
+ @trackers = other.trackers.dup
17
18
  end
18
-
19
- def category(category_field, options = {})
20
- if category_field.kind_of?(Symbol)
21
- track(:category, options.merge(:category => category_field))
22
- elsif category_field.kind_of?(Hash)
23
- track(:category, category_field.merge(options))
24
- end
19
+
20
+ # Drop all trackers
21
+ def reset!
22
+ @trackers = []
23
+ end
24
+
25
+ # Include missing trackers through method missing.
26
+ def method_missing(tracker_method, *args)
27
+ track(tracker_method, *args)
25
28
  end
26
29
 
27
- def duration(duration_field, options = {})
28
- if duration_field.kind_of?(Symbol)
29
- track(:duration, options.merge(:duration => duration_field))
30
- elsif duration_field.kind_of?(Hash)
31
- track(:duration, duration_field.merge(options))
32
- end
33
- end
34
-
35
- def track(tracker_klass, options = {})
36
- require "#{File.dirname(__FILE__)}/../tracker/#{tracker_klass}"
37
- tracker_klass = RequestLogAnalyzer::Tracker.const_get(tracker_klass.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')) if tracker_klass.kind_of?(Symbol)
30
+ # Helper function to initialize a tracker and add it to the tracker array.
31
+ # <tt>tracker_class</tt> The class to include
32
+ # <tt>optiont</tt> The options to pass to the trackers.
33
+ def track(tracker_klass, value_field = {}, other_options = {})
34
+ options = value_field.kind_of?(Symbol) ? other_options.merge(:value => value_field) : value_field.merge(other_options)
35
+ tracker_klass = RequestLogAnalyzer::Tracker.const_get(RequestLogAnalyzer::to_camelcase(tracker_klass)) if tracker_klass.kind_of?(Symbol)
38
36
  @trackers << tracker_klass.new(options)
39
37
  end
40
38
  end
41
-
39
+
42
40
  attr_reader :trackers
43
41
  attr_reader :warnings_encountered
44
-
42
+
43
+ # Initialize summarizer.
44
+ # Generate trackers from speciefied source.file_format.report_trackers and set them up
45
45
  def initialize(source, options = {})
46
46
  super(source, options)
47
47
  @warnings_encountered = {}
48
48
  @trackers = source.file_format.report_trackers
49
49
  setup
50
50
  end
51
-
51
+
52
52
  def setup
53
53
  end
54
-
54
+
55
+ # Call prepare on all trackers.
55
56
  def prepare
56
57
  raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
57
58
  @trackers.each { |tracker| tracker.prepare }
58
- end
59
-
59
+ end
60
+
61
+ # Pass all requests to trackers and let them update if necessary.
62
+ # <tt>request</tt> The request to pass.
60
63
  def aggregate(request)
61
64
  @trackers.each do |tracker|
62
65
  tracker.update(request) if tracker.should_update?(request)
63
66
  end
64
67
  end
65
-
68
+
69
+ # Call finalize on all trackers. Saves a YAML dump if this is set in the options.
66
70
  def finalize
67
71
  @trackers.each { |tracker| tracker.finalize }
72
+ save_results_dump(options[:yaml]) if options[:yaml]
73
+ end
74
+
75
+ # Saves the results of all the trackers in YAML format to a file.
76
+ # <tt>filename</tt> The file to store the YAML dump in.
77
+ def save_results_dump(filename)
78
+ File.open(filename, 'w') { |file| file.write(to_yaml) }
79
+ end
80
+
81
+ # Exports all the tracker results to YAML. It will call the to_yaml_object method
82
+ # for every tracker and combines these into a single YAML export.
83
+ def to_yaml
84
+ require 'yaml'
85
+ trackers_export = @trackers.inject({}) do |export, tracker|
86
+ export[tracker.title] = tracker.to_yaml_object; export
87
+ end
88
+ YAML::dump(trackers_export)
68
89
  end
69
-
70
- def report(output=STDOUT, report_width = 80, color = false)
71
- report_header(output, report_width, color)
90
+
91
+ # Call report on all trackers.
92
+ # <tt>output</tt> RequestLogAnalyzer::Output object to output to
93
+ def report(output)
94
+ report_header(output)
72
95
  if source.parsed_requests > 0
73
- @trackers.each { |tracker| tracker.report(output, report_width, color) }
96
+ @trackers.each { |tracker| output.report_tracker(tracker) }
74
97
  else
75
- output << "\n"
76
- output << "There were no requests analyzed.\n"
98
+ output.puts
99
+ output.puts('There were no requests analyzed.')
77
100
  end
78
- report_footer(output, report_width, color)
101
+ report_footer(output)
79
102
  end
80
-
81
- def report_header(output=STDOUT, report_width = 80, color = false)
82
- output << "Request summary\n"
83
- output << green("━" * report_width, color) + "\n"
84
- output << "Parsed lines: #{green(source.parsed_lines, color)}\n"
85
- output << "Parsed requests: #{green(source.parsed_requests, color)}\n"
86
- output << "Skipped lines: #{green(source.skipped_lines, color)}\n" if source.skipped_lines > 0
87
- if has_warnings?
88
- output << "Warnings: " + @warnings_encountered.map { |(key, value)| "#{key.inspect}: #{blue(value, color)}" }.join(', ') + "\n"
103
+
104
+ # Generate report header.
105
+ # <tt>output</tt> RequestLogAnalyzer::Output object to output to
106
+ def report_header(output)
107
+ output.title("Request summary")
108
+
109
+ output.with_style(:cell_separator => false) do
110
+ output.table({:width => 20}, {:font => :bold}) do |rows|
111
+ rows << ['Parsed lines:', source.parsed_lines]
112
+ rows << ['Skipped lines:', source.skipped_lines]
113
+ rows << ['Parsed requests:', source.parsed_requests]
114
+ rows << ['Skipped requests:', source.skipped_requests]
115
+ rows << ["Warnings:", @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
116
+ end
89
117
  end
90
118
  output << "\n"
91
119
  end
92
-
93
- def report_footer(output=STDOUT, report_width = 80, color = false)
94
- output << "\n"
95
- if has_serious_warnings?
96
- output << green("━" * report_width, color) + "\n"
97
- output << "Multiple warnings were encountered during parsing. Possibly, your logging " + "\n"
98
- output << "is not setup correctly. Visit this website for logging configuration tips:" + "\n"
99
- output << blue("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging", color) + "\n"
120
+
121
+ # Generate report footer.
122
+ # <tt>output</tt> RequestLogAnalyzer::Output object to output to
123
+ def report_footer(output)
124
+ if has_log_ordering_warnings?
125
+ output.title("Parse warnings")
126
+
127
+ output.puts "Parsable lines were encountered without a header line before it. It"
128
+ output.puts "could be that logging is not setup correctly for your application."
129
+ output.puts "Visit this website for logging configuration tips:"
130
+ output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")
131
+ output.puts
100
132
  end
101
133
  end
102
-
134
+
135
+ # Returns true if there were any warnings generated by the trackers
103
136
  def has_warnings?
104
- @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
137
+ @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
105
138
  end
106
-
107
- def has_serious_warnings?
108
- @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 10
139
+
140
+ # Returns true if there were any log ordering warnings
141
+ def has_log_ordering_warnings?
142
+ @warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
109
143
  end
110
-
144
+
145
+ # Store an encountered warning
146
+ # <tt>type</tt> Type of warning
147
+ # <tt>message</tt> Warning message
148
+ # <tt>lineno</tt> The line on which the error was encountered
111
149
  def warning(type, message, lineno)
112
150
  @warnings_encountered[type] ||= 0
113
151
  @warnings_encountered[type] += 1
@@ -1,44 +1,48 @@
1
1
  module RequestLogAnalyzer::Aggregator
2
2
 
3
+ def self.const_missing(const)
4
+ RequestLogAnalyzer::load_default_class_file(self, const)
5
+ end
6
+
3
7
  # The base class of an aggregator. This class provides the interface to which
4
8
  # every aggregator should comply (by simply subclassing this class).
5
9
  class Base
6
-
7
- include RequestLogAnalyzer::FileFormat::Awareness
8
-
9
- attr_reader :options
10
- attr_reader :source
10
+
11
+ attr_reader :options, :source
11
12
 
12
13
  # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
13
14
  # It will include the specific file format module.
14
15
  def initialize(source, options = {})
15
16
  @source = source
16
- self.register_file_format(source.file_format)
17
17
  @options = options
18
18
  end
19
19
 
20
- # The prepare function is called just before parsing starts. This function
20
+ # The prepare function is called just before parsing starts. This function
21
21
  # can be used to initialie variables, etc.
22
22
  def prepare
23
23
  end
24
-
24
+
25
25
  # The aggregate function is called for every request.
26
26
  # Implement the aggregating functionality in this method
27
27
  def aggregate(request)
28
28
  end
29
-
29
+
30
30
  # The finalize function is called after all sources are parsed and no more
31
31
  # requests will be passed to the aggregator
32
32
  def finalize
33
33
  end
34
-
34
+
35
35
  # The warning method is called if the parser eits a warning.
36
36
  def warning(type, message, lineno)
37
- end
38
-
37
+ end
38
+
39
39
  # The report function is called at the end. Implement any result reporting
40
40
  # in this function.
41
- def report(output = STDOUT, report_width = 80, color = false)
41
+ def report(output)
42
+ end
43
+
44
+ # The source_change function gets called when handling a source is started or finished.
45
+ def source_change(change, filename)
42
46
  end
43
47
 
44
48
  end