request-log-analyzer 1.4.0 → 1.5.0

data/lib/request_log_analyzer/source/log_parser.rb

@@ -20,6 +20,7 @@ module RequestLogAnalyzer::Source
     PARSE_STRATEGIES = ['cautious', 'assume-correct']
 
     attr_reader :source_files, :current_file, :current_lineno
+    attr_reader :warnings, :parsed_lines, :parsed_requests, :skipped_lines, :skipped_requests
 
     # Initializes the log file parser instance.
     # It will apply the language specific FileFormat module to this instance. It will use the line
@@ -29,6 +30,7 @@ module RequestLogAnalyzer::Source
     # <tt>options</tt>:: A hash of options that are used by the parser
     def initialize(format, options = {})
       super(format, options)
+      @warnings         = 0
       @parsed_lines     = 0
       @parsed_requests  = 0
       @skipped_lines    = 0
@@ -142,21 +144,27 @@ module RequestLogAnalyzer::Source
     # <tt>io</tt>:: The IO instance to use as source
     # <tt>options</tt>:: A hash of options that can be used by the parser.
     def parse_io(io, options = {}, &block) # :yields: request
-      @current_lineno = 1
+      @current_lineno = 0
       while line = io.gets
-        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
-
-        if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
-          @parsed_lines += 1
-          update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
-        end
-
         @current_lineno += 1
+        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
+        parse_line(line, &block)
       end
 
       warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
       @current_lineno = nil
     end
+    
+    # Parses a single line using the current file format. If successful, use the parsed
+    # information to build a request
+    # <tt>line</tt>:: The line to parse
+    # <tt>block</tt>:: The block to send fully parsed requests to.
+    def parse_line(line, &block) # :yields: request
+      if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
+        @parsed_lines += 1
+        update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
+      end
+    end
 
     # Add a block to this method to install a progress handler while parsing.
     # <tt>proc</tt>:: The proc that will be called to handle progress update messages
@@ -186,6 +194,7 @@ module RequestLogAnalyzer::Source
     # <tt>type</tt>:: The warning type (a Symbol)
     # <tt>message</tt>:: A message explaining the warning
     def warn(type, message)
+      @warnings += 1
       @warning_handler.call(type, message, @current_lineno) if @warning_handler
     end
 
@@ -214,7 +223,13 @@ module RequestLogAnalyzer::Source
     #
     # <tt>request_data</tt>:: A hash of data that was parsed from the last line.
     def update_current_request(request_data, &block) # :yields: request
-      if header_line?(request_data)
+      if alternative_header_line?(request_data)
+        if @current_request
+          @current_request << request_data
+        else
+          @current_request = @file_format.request(request_data)
+        end
+      elsif header_line?(request_data)
         if @current_request
           case options[:parse_strategy]
           when 'assume-correct'
@@ -258,10 +273,18 @@ module RequestLogAnalyzer::Source
       @skipped_requests += 1 unless accepted
     end
 
+
+    # Checks whether a given line hash is an alternative header line according to the current file format.
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
+    def alternative_header_line?(hash)
+      hash[:line_definition].header == :alternative
+    end
+
+
     # Checks whether a given line hash is a header line according to the current file format.
     # <tt>hash</tt>:: A hash of data that was parsed from the line.
     def header_line?(hash)
-      hash[:line_definition].header
+      hash[:line_definition].header == true
     end
 
     # Checks whether a given line hash is a footer line  according to the current file format.

data/lib/request_log_analyzer/tracker/count.rb

@@ -0,0 +1,93 @@
+module RequestLogAnalyzer::Tracker
+
+  # Get highest count of a specific attribute
+  class Count < Base
+
+    include RequestLogAnalyzer::Tracker::StatisticsTracking
+
+    attr_reader :categories
+
+    def prepare
+      raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
+      raise "No field to count has been set up #{self.inspect}" unless options[:field]
+
+      @categorizer = create_lambda(options[:category])
+      @counter     = create_lambda(options[:field])
+      @categories  = {}
+    end
+    
+    # Get the duration information fron the request and store it in the different categories.
+    # <tt>request</tt> The request.
+    def update(request)
+      category = @categorizer.call(request)
+      count    = @counter.call(request)
+      update_statistics(category, count) if count.kind_of?(Numeric) && !category.nil?
+    end
+
+    def report(output)
+      sortings = output.options[:sort] || [:sum, :mean]
+
+      sortings.each do |report|
+        case report
+        when :mean
+          report_table(output, :mean,   :title => "#{title} - sorted by mean")
+        when :stddev
+          report_table(output, :stddev, :title => "#{title} - sorted by standard deviation")
+        when :sum
+          report_table(output, :sum,    :title => "#{title} - sorted by sum")
+        when :hits
+          report_table(output, :hits,   :title => "#{title} - sorted by hits")
+        else
+          raise "Unknown duration report specified: #{report}!"
+        end
+      end
+    end
+    
+    # Block function to build a result table using a provided sorting function.
+    # <tt>output</tt> The output object.
+    # <tt>amount</tt> The number of rows in the report table (default 10).
+    # === Options
+    #  * </tt>:title</tt> The title of the table
+    #  * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
+    def report_table(output, sort, options = {}, &block)
+      output.puts
+
+      top_categories = output.slice_results(sorted_by(sort))
+      output.with_style(:top_line => true) do      
+        output.table(*statistics_header(:title => options[:title],:highlight => sort)) do |rows|
+          top_categories.each { |(cat, info)| rows.push(statistics_row(cat)) }
+        end
+      end
+      output.puts
+    end
+        
+    # Format an int to a nice string with decimal seperation.
+    # Also see http://en.wikipedia.org/wiki/Terabyte
+    # <tt>count</tt> the value to reduce
+    def display_value(count)
+      return "- " if count.nil?
+      return "0 " if count.zero?
+
+      case Math.log10(count).floor
+        when  1...4  then '%d ' % count
+        when  4...7  then '%dk' % (count / 1000)
+        when  7...10 then '%dM' % (count / 1000_000)
+        when 10...13 then '%dG' % (count / 1000_000_000)
+        when 13...16 then '%dT' % (count / 1000_000_000_000)
+        else              '%dP' % (count / 1000_000_000_000_000)
+      end
+
+    end
+
+    # Returns the title of this tracker for reports
+    def title
+      options[:title]  || 'Total'
+    end
+
+    # Returns all the categories and the tracked duration as a hash than can be exported to YAML
+    def to_yaml_object
+      return nil if @categories.empty?
+      @categories
+    end
+  end
+end

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
 
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.4.0"
+  VERSION = "1.5.0"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.4.0"
-  s.date    = "2009-09-30"
+  s.version = "1.5.0"
+  s.date    = "2009-11-18"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -26,8 +26,8 @@ Gem::Specification.new do |s|
 
   s.requirements << "To use the database inserter, ActiveRecord and an appropriate database adapter are required."
 
-  s.add_development_dependency('rspec', '>= 1.2.4')
-  s.add_development_dependency('git',   '>= 1.1.0')
+  s.add_development_dependency('rspec',    '>= 1.2.4')
+  s.add_development_dependency('git',      '>= 1.1.0')
 
   s.authors  = ['Willem van Bergen', 'Bart ten Brinke']
   s.email    = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
@@ -35,6 +35,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/lib/macros.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log lib/request_log_analyzer/file_format/amazon_s3.rb tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/database.rb spec/fixtures/decompression.log.bz2 spec/fixtures/rails_unordered.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/testing_format.rb spec/fixtures/test_order.log spec/fixtures/rails.db lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/aggregator.rb spec/unit/request_spec.rb lib/cli/progressbar.rb lib/request_log_analyzer/mailer.rb README.rdoc lib/request_log_analyzer/database/warning.rb spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/mailer_spec.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb spec/database.yml Rakefile lib/request_log_analyzer/database/connection.rb spec/unit/filter/filter_spec.rb spec/fixtures/test_language_combined.log lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb lib/cli/database_console.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/lib/mocks.rb spec/lib/helpers.rb spec/fixtures/rails_1x.log lib/cli/database_console_init.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/fixtures/decompression.log.zip spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log tasks/github-gem.rake spec/unit/database/database_spec.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/traffic.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/syslog_1x.log spec/fixtures/rails_22.log lib/request_log_analyzer/database/request.rb spec/fixtures/multiple_files_2.log LICENSE lib/request_log_analyzer/source/database_loader.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
-  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/request_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/mailer_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
+  s.files      = %w(lib/request_log_analyzer/output/html.rb spec/unit/tracker/hourly_spread_spec.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/unit/tracker/count_tracker_spec.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/integration/munin_plugins_rails_spec.rb spec/lib/helpers.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb spec/fixtures/test_language_combined.log lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml spec/unit/request_spec.rb lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/tracker/count.rb lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log lib/request_log_analyzer/file_format/rails.rb Rakefile spec/unit/mailer_spec.rb lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
+  s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/filter/filter_spec.rb spec/unit/tracker/count_tracker_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
 end

data/spec/fixtures/mysql_slow_query.log

@@ -0,0 +1,110 @@
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:13:56
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:14:07
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 9  Lock_time: 0  Rows_sent: 258841  Rows_examined: 258841
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `sessions`;
+# Time: 091112 18:14:44
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 6  Lock_time: 0  Rows_sent: 603732  Rows_examined: 603732
+use sensire_s;
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:14:56
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 12  Lock_time: 0  Rows_sent: 1068438  Rows_examined: 1068438
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:16:18
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 17  Lock_time: 0  Rows_sent: 1822689  Rows_examined: 1822689
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:16:28
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 10  Lock_time: 0  Rows_sent: 1113903  Rows_examined: 1113903
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:17:37
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 19  Lock_time: 0  Rows_sent: 1631168  Rows_examined: 1631168
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:17:49
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 11  Lock_time: 0  Rows_sent: 951103  Rows_examined: 951103
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:19:31
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 7  Lock_time: 0  Rows_sent: 747129  Rows_examined: 747129
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:19:49
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 17  Lock_time: 0  Rows_sent: 1904679  Rows_examined: 1904679
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:21:34
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 59  Lock_time: 0  Rows_sent: 5013829  Rows_examined: 5013829
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:21:54
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 20  Lock_time: 0  Rows_sent: 1634467  Rows_examined: 1634467
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:22:58
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 5  Lock_time: 0  Rows_sent: 634683  Rows_examined: 634683
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:23:58
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 5  Lock_time: 0  Rows_sent: 444697  Rows_examined: 444697
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:24:34
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 7  Lock_time: 0  Rows_sent: 669623  Rows_examined: 669623
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:26:18
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 27  Lock_time: 0  Rows_sent: 2727395  Rows_examined: 2727395
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:26:28
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 10  Lock_time: 0  Rows_sent: 859164  Rows_examined: 859164
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:27:50
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 29  Lock_time: 0  Rows_sent: 2607634  Rows_examined: 2607634
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 18:27:59
+# User@Host: admin[admin] @ db1 [10.0.0.1]
+# Query_time: 9  Lock_time: 0  Rows_sent: 766241  Rows_examined: 766241
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 18:30:39
+# User@Host: user_1[user_1] @ rails1 [10.0.1.3]
+# Query_time: 6  Lock_time: 0  Rows_sent: 0  Rows_examined: 1648536
+SELECT * FROM `clients` WHERE (1=1 
+                              AND clients.valid_from < '2009-12-05' AND (clients.valid_to IS NULL or clients.valid_to > '2009-11-20')
+                               AND clients.index > 0
+                              ) AND (clients.deleted_at IS NULL);
+# Time: 091112 20:15:16
+# User@Host: user_2[user_2] @ rails1 [10.0.1.3]
+# Query_time: 7  Lock_time: 0  Rows_sent: 796074  Rows_examined: 796074
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 20:15:29
+# User@Host: user_2[user_2] @ rails1 [10.0.1.3]
+# Query_time: 12  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 20:15:45
+# User@Host: user_2[user_2] @ rails1 [10.0.1.3]
+# Query_time: 12  Lock_time: 0  Rows_sent: 259350  Rows_examined: 259350
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `sessions`;
+# Time: 091112 20:18:50
+# User@Host: user_1[user_1] @ rails1 [10.0.1.3]
+# Query_time: 28  Lock_time: 0  Rows_sent: 2727404  Rows_examined: 2727404
+use zuwe_p;
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
+# Time: 091112 20:19:18
+# User@Host: user_1[user_1] @ rails1 [10.0.1.3]
+# Query_time: 28  Lock_time: 0  Rows_sent: 859164  Rows_examined: 859164
+SELECT /*!40001 SQL_NO_CACHE */ * FROM `events`;
+# Time: 091112 20:19:40
+# User@Host: user_1[user_1] @ db1 [10.1.1.178]
+# Query_time: 6  Lock_time: 0  Rows_sent: 33  Rows_examined: 1648542
+SELECT * FROM `clients` WHERE (valid_to IS NULL OR valid_to > '2009-12-12') AND ((`clients`.`deleted` = 1) AND (clients.deleted_at IS NULL));

data/spec/integration/command_line_usage_spec.rb

@@ -72,9 +72,9 @@ describe RequestLogAnalyzer, 'running from command line' do
   end
 
   it "should dump the results to a YAML file" do
-    run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:dump)}")
-    File.exist?(temp_output_file(:dump)).should be_true
-    YAML::load(File.read(temp_output_file(:dump))).should have_at_least(1).item
+    run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:yaml)}")
+    File.exist?(temp_output_file(:yaml)).should be_true
+    YAML::load(File.read(temp_output_file(:yaml))).should have_at_least(1).item
   end
 
   it "should parse 4 requests from the standard input" do

data/spec/integration/mailer_spec.rb

@@ -0,0 +1,179 @@
+require File.dirname(__FILE__) + '/../spec_helper.rb'
+require 'socket'
+
+describe RequestLogAnalyzer, 'running mailer integration' do
+
+  before(:each) do
+    @log_file = temp_output_file('mailtrap.log')
+
+    Process.fork {
+      Mailtrap.new('localhost', 2525, true, @log_file)
+      Process.exit! # Do not call rspec after exit hook!
+    }
+  end
+
+  after(:each) do
+    cleanup_temp_files!
+  end
+
+  it "should send plaintext mail" do
+    RequestLogAnalyzer::Controller.build(
+      :mail         => 'root@localhost',
+      :mailhost     => 'localhost:2525',
+      :source_files => log_fixture(:rails_1x),
+      :format       => RequestLogAnalyzer::FileFormat::Rails,
+      :no_progress  => true
+    ).run!
+    
+    Process.wait # Wait for mailer to complete
+
+    find_string_in_file("From: <contact@railsdoctors.com>", @log_file).should_not be_nil
+    find_string_in_file("To: <root@localhost>", @log_file).should_not be_nil
+    find_string_in_file("From: Request-log-analyzer reporter <contact@railsdoctors.com>", @log_file).should_not be_nil
+    find_string_in_file("Request summary", @log_file).should_not be_nil
+    find_string_in_file("PeopleController#show.html [ |    1 |  0.29s |  0.29s |  0.00s |  0.29s |  0.29s", @log_file).should_not be_nil
+  end
+
+  it "should send html mail" do    
+    RequestLogAnalyzer::Controller.build(
+      :output       => 'HTML',
+      :mail         => 'root@localhost',
+      :mailhost     => 'localhost:2525',
+      :source_files => log_fixture(:rails_1x),
+      :format       => RequestLogAnalyzer::FileFormat::Rails,
+      :no_progress  => true
+    ).run!
+  
+    Process.wait # Wait for mailer to complete
+
+    find_string_in_file("From: <contact@railsdoctors.com>", @log_file).should_not be_nil
+    find_string_in_file("To: <root@localhost>", @log_file).should_not be_nil
+    find_string_in_file("From: Request-log-analyzer reporter <contact@railsdoctors.com>", @log_file).should_not be_nil
+    find_string_in_file('<h1>Request-log-analyzer summary report</h1>', @log_file).should_not be_nil
+    find_string_in_file('<td class="alt">0.29s</td></tr><tr><td>DashboardController#index.html [GET]</td>', @log_file).should_not be_nil
+  end
+end
+
+
+
+# Mailtrap
+#     by Matt Mower <self@mattmower.com>
+#     http://matt.blogs.it/
+#
+# Included in RLA because original mailtrap puts anoying stuff when called
+# through ruby.
+# 
+# Mailtrap creates a TCP server that listens on a specified port for SMTP
+# clients. Accepts the connection and talks just enough of the SMTP protocol
+# for them to deliver a message which it writes to disk.
+#
+class Mailtrap
+  VERSION = '0.2.1'
+
+  # Create a new Mailtrap on the specified host:port. If once it true it
+  # will listen for one message then exit. Specify the msgdir where messages
+  # are written.
+  def initialize( host, port, once, msgfile )
+    @host = host
+    @port = port
+    @once = once
+    @msgfile = msgfile
+
+    File.open( @msgfile, "a" ) do |file|
+      file.puts "\n* Mailtrap started at #{@host}:#{port}\n"
+    end
+
+    service = TCPServer.new( @host, @port )
+    accept( service )
+  end
+
+  # Service one or more SMTP client connections
+  def accept( service )
+    while session = service.accept
+
+      class << session
+        def get_line
+          line = gets
+          line.chomp! unless line.nil?
+          line          
+        end
+      end
+
+      begin
+        serve( session )
+      rescue Exception => e
+      end
+
+      break if @once
+    end    
+  end
+
+  # Write a plain text dump of the incoming email to a text
+  # file. The file will be in the @msgdir folder and will
+  # be called smtp0001.msg, smtp0002.msg, and so on.
+  def write( from, to_list, message )
+
+    # Strip SMTP commands from To: and From:
+    from.gsub!( /MAIL FROM:\s*/, "" )
+    to_list = to_list.map { |to| to.gsub( /RCPT TO:\s*/, "" ) }
+
+    # Append to the end of the messages file
+    File.open( @msgfile, "a" ) do |file|
+      file.puts "* Message begins"
+      file.puts "From: #{from}"
+      file.puts "To: #{to_list.join(", ")}"
+      file.puts "Body:"
+      file.puts message
+      file.puts "\n* Message ends"
+    end
+
+  end
+
+  # Talk pidgeon-SMTP to the client to get them to hand over the message
+  # and go away.
+  def serve( connection )
+    connection.puts( "220 #{@host} MailTrap ready ESTMP" )
+    helo = connection.get_line # whoever they are
+
+    if helo =~ /^EHLO\s+/
+      connection.puts "250-#{@host} offers just ONE extension my pretty"
+      connection.puts "250 HELP"
+    end
+
+    # Accept MAIL FROM:
+    from = connection.get_line
+    connection.puts( "250 OK" )
+
+    to_list = []
+
+    # Accept RCPT TO: until we see DATA
+    loop do
+      to = connection.get_line
+      break if to.nil?
+
+      if to =~ /^DATA/
+        connection.puts( "354 Start your message" )
+        break
+      else
+        to_list << to
+        connection.puts( "250 OK" )
+      end
+    end
+
+    # Capture the message body terminated by <CR>.<CR>
+    lines = []
+    loop do
+      line = connection.get_line
+      break if line.nil? || line == "."
+      lines << line
+    end
+
+    # We expect the client will go away now
+    connection.puts( "250 OK" )
+    connection.gets # Quit
+    connection.puts "221 Seeya"
+    connection.close
+
+    write( from, to_list, lines.join( "\n" ) )
+  end
+end

data/spec/integration/munin_plugins_rails_spec.rb

@@ -0,0 +1,58 @@
+require File.dirname(__FILE__) + '/../spec_helper.rb'
+
+describe RequestLogAnalyzer, 'when harvesting like munin-plugins-rails the YAML output' do
+  
+  before(:each) do 
+    cleanup_temp_files!    
+    run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:yaml)}")
+    @rla = YAML::load(File.read(temp_output_file(:yaml)))
+  end
+  
+  after(:each) do
+    cleanup_temp_files!
+  end
+  
+  it "should contain database times" do
+    @rla["Database time"].each do |item|
+      item[1][:min].should_not be_nil
+      item[1][:max].should_not be_nil
+      item[1][:hits].should_not be_nil
+      item[1][:sum].should_not be_nil
+    end
+  end
+
+  it "should contain request times" do
+    @rla["Request duration"].each do |item|
+      item[1][:min].should_not be_nil
+      item[1][:max].should_not be_nil
+      item[1][:hits].should_not be_nil
+      item[1][:sum].should_not be_nil
+    end
+  end
+
+  it "should contain failed requests" do
+    @rla.keys.should include("Failed requests")
+  end
+
+  it "should contain Process blockers" do
+    @rla.keys.should include("Process blockers (> 1 sec duration)")
+  end
+
+  it "should contain HTTP Methods" do
+    @rla["HTTP methods"]["GET"].should_not be_nil
+  end
+
+  it "should contain HTTP Methods" do
+    @rla["HTTP methods"]["GET"].should_not be_nil
+  end
+
+  it "should contain view rendering times" do
+    @rla["View rendering time"].each do |item|
+      item[1][:min].should_not be_nil
+      item[1][:max].should_not be_nil
+      item[1][:hits].should_not be_nil
+      item[1][:sum].should_not be_nil
+    end
+  end
+
+end