request-log-analyzer 1.10.1 → 1.11.0

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -29,6 +29,13 @@ module RequestLogAnalyzer::FileFormat
       line.capture(:action)
       line.capture(:format)
     end
+
+    # Parameters: {"action"=>"cached", "controller"=>"cached"}
+    line_definition :parameters do |line|
+      line.teaser   = /  Parameters:/
+      line.regexp   = /  Parameters:\s+(\{.*\})/
+      line.capture(:params).as(:eval)
+    end
     
     # Completed 200 OK in 224ms (Views: 200.2ms | ActiveRecord: 3.4ms)
     # Completed 302 Found in 23ms

data/lib/request_log_analyzer/filter.rb

@@ -1,9 +1,5 @@
 module RequestLogAnalyzer::Filter
 
-  autoload :Field,     'request_log_analyzer/filter/field'
-  autoload :Timespan,  'request_log_analyzer/filter/timespan'
-  autoload :Anonymize, 'request_log_analyzer/filter/anonymize'
-
   # Base filter class used to filter input requests.
   # All filters should interit from this base.
   class Base
@@ -24,5 +20,8 @@ module RequestLogAnalyzer::Filter
       request
     end
   end
+end
 
-end
+require 'request_log_analyzer/filter/field'
+require 'request_log_analyzer/filter/timespan'
+require 'request_log_analyzer/filter/anonymize'

data/lib/request_log_analyzer/line_definition.rb

@@ -70,6 +70,10 @@ module RequestLogAnalyzer
       captures << new_capture_hash
       CaptureDefiner.new(new_capture_hash)
     end
+    
+    def all_captured_variables
+      captures.map { |c| c[:name] } + captures.map { |c| c[:provides] }.compact.map { |pr| pr.keys }.flatten
+    end
 
     # Checks whether a given line matches this definition.
     # It will return false if a line does not match. If the line matches, a hash is returned
@@ -128,9 +132,7 @@ module RequestLogAnalyzer
 
     # Returns true if this line captures values of the given name
     def captures?(name)
-      captures.any? { |c| c[:name] == name }
+      all_captured_variables.include?(name)
     end
-
   end
-
-end
+end

data/lib/request_log_analyzer/output.rb

@@ -1,10 +1,6 @@
 # Module for generating output
 module RequestLogAnalyzer::Output
 
-  autoload :FixedWidth, 'request_log_analyzer/output/fixed_width'
-  autoload :HTML,       'request_log_analyzer/output/html'
-  autoload :FancyHTML,  'request_log_analyzer/output/fancy_html'
-  
   # Loads a Output::Base subclass instance.
   def self.load(file_format, *args)
     
@@ -111,6 +107,8 @@ module RequestLogAnalyzer::Output
     def table_has_header?(columns)
       columns.any? { |column| !column[:title].nil? }
     end
-
   end
 end
+
+require 'request_log_analyzer/output/fixed_width'
+require 'request_log_analyzer/output/html'

data/lib/request_log_analyzer/source.rb

@@ -8,9 +8,6 @@
 # - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
 module RequestLogAnalyzer::Source
 
-  autoload :LogParser,      'request_log_analyzer/source/log_parser'
-  autoload :DatabaseLoader, 'request_log_analyzer/source/database_loader'
-
   # The base Source class. All other sources should inherit from this class.
   #
   # A source implememtation should at least implement the each_request method, which should yield
@@ -65,4 +62,6 @@ module RequestLogAnalyzer::Source
     end
 
   end
-end
+end
+
+require 'request_log_analyzer/source/log_parser'

data/lib/request_log_analyzer/source/log_parser.rb

@@ -13,6 +13,11 @@ module RequestLogAnalyzer::Source
 
     include Enumerable
 
+    # The maximum number of bytes to read from a line.
+    DEFAULT_MAX_LINE_LENGTH = 8096
+    
+    DEFAULT_LINE_DIVIDER = "\n"
+
     # The default parse strategy that will be used to parse the input.
     DEFAULT_PARSE_STRATEGY = 'assume-correct'
 
@@ -48,6 +53,14 @@ module RequestLogAnalyzer::Source
       end
     end
 
+    def max_line_length
+      file_format.max_line_length || DEFAULT_MAX_LINE_LENGTH
+    end
+    
+    def line_divider
+      file_format.line_divider || DEFAULT_LINE_DIVIDER
+    end
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
@@ -132,6 +145,13 @@ module RequestLogAnalyzer::Source
     def parse_stream(stream, options = {}, &block)
       parse_io(stream, options, &block)
     end
+    
+    # Parses a string. It will simply call parse_io. This function does not support progress updates.
+    # <tt>string</tt>:: The string that should be parsed.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def parse_string(string, options = {}, &block)
+      parse_io(StringIO.new(string), options, &block)
+    end
 
     # This method loops over each line of the input stream. It will try to parse this line as any of
     # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
@@ -143,11 +163,15 @@ module RequestLogAnalyzer::Source
     # - The method will yield progress updates if a progress handler is installed using progress=
     # - The method will yield parse warnings if a warning handler is installed using warning=
     #
+    # This is a Ruby 1.9 specific version that offers memory protection.
+    #
     # <tt>io</tt>:: The IO instance to use as source
     # <tt>options</tt>:: A hash of options that can be used by the parser.
-    def parse_io(io, options = {}, &block) # :yields: request
-      @current_lineno = 0
-      while line = io.gets
+    def parse_io_19(io, options = {}, &block) # :yields: request
+      @max_line_length = options[:max_line_length] || max_line_length
+      @line_divider    = options[:line_divider]    || line_divider
+      @current_lineno  = 0
+      while line = io.gets(@line_divider, @max_line_length)
         @current_lineno += 1
         @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
         parse_line(line, &block)
@@ -157,6 +181,35 @@ module RequestLogAnalyzer::Source
       @current_lineno = nil
     end
     
+    # This method loops over each line of the input stream. It will try to parse this line as any of
+    # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
+    # It will then combine these parsed line into requests using heuristics. These requests (see
+    # RequestLogAnalyzer::Request) will then be yielded for further processing in the pipeline.
+    #
+    # - RequestLogAnalyzer::LineDefinition#matches is called to test if a line matches a line definition of the file format.
+    # - update_current_request is used to combine parsed lines into requests using heuristics.
+    # - The method will yield progress updates if a progress handler is installed using progress=
+    # - The method will yield parse warnings if a warning handler is installed using warning=
+    #
+    # This is a Ruby 1.8 specific version that doesn't offer memory protection.
+    #
+    # <tt>io</tt>:: The IO instance to use as source
+    # <tt>options</tt>:: A hash of options that can be used by the parser.    
+    def parse_io_18(io, options = {}, &block) # :yields: request
+      @line_divider    = options[:line_divider]    || line_divider
+      @current_lineno  = 0
+      while line = io.gets(@line_divider)
+        @current_lineno += 1
+        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
+        parse_line(line, &block)
+      end
+
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+      @current_lineno = nil
+    end
+    
+    alias_method :parse_io, RUBY_VERSION.to_f < 1.9 ? :parse_io_18 : :parse_io_19
+    
     # Parses a single line using the current file format. If successful, use the parsed
     # information to build a request
     # <tt>line</tt>:: The line to parse
@@ -295,5 +348,4 @@ module RequestLogAnalyzer::Source
       hash[:line_definition].footer
     end
   end
-
 end

data/lib/request_log_analyzer/tracker.rb

@@ -1,12 +1,5 @@
 module RequestLogAnalyzer::Tracker
 
-  autoload :Duration,     'request_log_analyzer/tracker/duration'
-  autoload :Frequency,    'request_log_analyzer/tracker/frequency'
-  autoload :HourlySpread, 'request_log_analyzer/tracker/hourly_spread'
-  autoload :NumericValue, 'request_log_analyzer/tracker/numeric_value'
-  autoload :Timespan,     'request_log_analyzer/tracker/timespan'
-  autoload :Traffic,      'request_log_analyzer/tracker/traffic'
-
   # Base Tracker class. All other trackers inherit from this class
   #
   # Accepts the following options:
@@ -99,4 +92,11 @@ module RequestLogAnalyzer::Tracker
       nil
     end
   end
-end
+end
+
+require 'request_log_analyzer/tracker/numeric_value'
+require 'request_log_analyzer/tracker/duration'
+require 'request_log_analyzer/tracker/frequency'
+require 'request_log_analyzer/tracker/hourly_spread'
+require 'request_log_analyzer/tracker/timespan'
+require 'request_log_analyzer/tracker/traffic'

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.10.1"
-  s.date    = "2011-03-17"
+  s.version = "1.11.0"
+  s.date    = "2011-04-27"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -39,6 +39,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(.gitignore .infinity_test DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/mixins/gets_memory_protection.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fancy_html.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
+  s.files      = %w(.gitignore .infinity_test DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
   s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
 end

data/spec/fixtures/mysql_slow_query.log

@@ -1,4 +1,3 @@
-SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
 # Time: 091112 18:13:56
 # User@Host: admin[admin] @ db1 [10.0.0.1]
 # Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307

data/spec/integration/command_line_usage_spec.rb

@@ -14,11 +14,6 @@ describe RequestLogAnalyzer, 'running from command line' do
     output = run("#{log_fixture(:rails_1x)}")
     output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
   end
-
-  it "should function correctly with the gets mixin" do
-    output = run("#{log_fixture(:rails_1x)} --gets-memory-protection")
-    output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
-  end
   
   it "should find 2 requests when parsing a compressed file" do
     output = run("#{log_fixture(:decompression, :tgz)}")

data/spec/lib/helpers.rb

@@ -11,8 +11,8 @@ module RequestLogAnalyzer::RSpec::Helpers
   end
 
   # Creates a log file given some lines
-  def log_stream(*lines)
-    StringIO.new(lines.join("\n") + "\n")
+  def log_snippet(*lines)
+    StringIO.new(lines.join("\n") << "\n")
   end
 
   # Request loopback

data/spec/lib/matchers.rb

@@ -7,15 +7,23 @@ module RequestLogAnalyzer::RSpec::Matchers
       @captures  = []
     end
 
-    def capturing(*captures)
+    def and_capture(*captures)
       @captures += captures
       return self
     end
+    
+    alias_method :capturing, :and_capture
+    
+    def description
+      description = "have a #{@line_type.inspect} line definition"
+      description << " that captures #{@captures.join(', ')}" unless @captures.empty?
+      description
+    end
 
     def matches?(file_format)
       file_format = file_format.create if file_format.kind_of?(Class)
       if ld = file_format.line_definitions[@line_type]
-        @captures.all? { |c| ld.captures.map { |cd| cd[:name] }.include?(c) }
+        @captures.all? { |c| ld.all_captured_variables.include?(c) }
       else
         false
       end
@@ -24,10 +32,25 @@ module RequestLogAnalyzer::RSpec::Matchers
 
   class ParseLine
     
-    def initialize(line)
+    def initialize(line, line_description = nil)
       @line      = line
       @captures  = {}
       @line_type = nil
+      @line_description = line_description
+    end
+
+    def line_description
+      @full_line_description ||= if @line_description
+        if @line_type && @line_description =~ /^(?:with|without|having|using) /
+          "a #{@line_type.inspect} line #{@line_description}"
+        else
+          @line_description
+        end
+      elsif @line_type 
+        "a #{@line_type.inspect} line"
+      else
+        "line #{@line.inspect}"
+      end
     end
 
     def failure_message
@@ -43,11 +66,20 @@ module RequestLogAnalyzer::RSpec::Matchers
       @captures = captures
       return self
     end
+    
+    alias_method :capturing, :and_capture
 
     def fail(message)
       @failure_message = message
       return false
     end
+    
+    def description
+      description = "parse #{line_description}"
+      description << " as line type #{@line_type.inspect}" if @line_type
+      description << " and capture #{@captures.keys.join(', ')} correctly" unless @captures.empty?
+      description
+    end
 
     def matches?(file_format)
       if @line_hash = file_format.parse_line(@line)
@@ -70,8 +102,7 @@ module RequestLogAnalyzer::RSpec::Matchers
     return HasLineDefinition.new(line_type)
   end
 
-  def parse_line(line)
-    ParseLine.new(line)
+  def parse_line(line, line_description = nil)
+    ParseLine.new(line, line_description)
   end
-
-end
+end

data/spec/lib/mocks.rb

@@ -70,8 +70,4 @@ module RequestLogAnalyzer::RSpec::Mocks
     connection.stub!(:table_creator).and_return(table_creator)
     return connection
   end
-
-  def request_counter
-    @request_counter ||= mock('aggregator to count request')
-  end
-end
+end

data/spec/unit/database/base_class_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'request_log_analyzer/database'
 
 describe RequestLogAnalyzer::Database::Base do
 

data/spec/unit/file_format/amazon_s3_format_spec.rb

@@ -1,67 +1,70 @@
 require 'spec_helper'
 
 describe RequestLogAnalyzer::FileFormat::AmazonS3 do
+  
+  subject { RequestLogAnalyzer::FileFormat.load(:amazon_s3) }
 
-  before(:each) do
-    @file_format = RequestLogAnalyzer::FileFormat.load(:amazon_s3)
-    @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
-    @sample = '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"'
-  end
+  it { should be_well_formed }
+  it { should have_line_definition(:access).capturing(:bucket_owner, :bucket, :timestamp, :remote_ip, :requester,
+        :key, :operation, :total_time, :turnaround_time, :bytes_sent, :object_size, :referer, :user_agent) }
 
-  it "should be a valid file format" do
-    @file_format.should be_valid
-  end
+  it { should have(7).report_trackers }
 
-  it "should parse access lines and capture all of its fields" do
-    @file_format.should have_line_definition(:access).capturing(:bucket_owner, :bucket, :timestamp, :remote_ip, :requester,
-          :key, :operation, :total_time, :turnaround_time, :bytes_sent, :object_size, :referer, :user_agent)
-  end
+  let(:sample_get)  { '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"' }
+  let(:sample_copy) { '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -' }
+  
+  describe '#parse_line' do
+    it { should parse_line(sample_get, 'a GET line').and_capture(
+              :bucket_owner    => '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1',
+              :bucket          => 'sample-bucket',
+              :timestamp       => 20061006014214,
+              :remote_ip       => '207.171.172.6',
+              :key             => 'object.png',
+              :operation       => 'REST.GET.OBJECT',
+              :requester       => '65a011a29cdf8ec533ec3d1ccaae921c',
+              :request_id      => 'C980091AD89C936A',
+              :request_uri     => 'GET /sample-bucket/object.png HTTP/1.1',
+              :error_code      => nil,
+              :http_status     => 200,
+              :total_time      => 0.988,
+              :turnaround_time => 0.987,
+              :bytes_sent      => 1243,
+              :object_size     => 1243,
+              :user_agent      => 'aranhabot',
+              :referer         => nil) 
+    }
 
-  it "should match the sample line" do
-    @file_format.parse_line(@sample).should include(:line_definition, :captures)
-  end
+    it { should parse_line(sample_copy, 'a COPY line').and_capture(
+              :bucket_owner    => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
+              :bucket          => 'testbucket',
+              :timestamp       => 20100303230459,
+              :remote_ip       => '174.119.31.76',
+              :key             => 'files/image.png',
+              :operation       => 'REST.COPY.OBJECT',
+              :requester       => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
+              :request_id      => 'ACCC34B843C87BC9',
+              :request_uri     => 'PUT /files/image.png HTTP/1.1',
+              :error_code      => nil,
+              :http_status     => 200,
+              :total_time      => 0.365,
+              :turnaround_time => 0.319,
+              :bytes_sent      => 234,
+              :object_size     => 65957,
+              :user_agent      => '',
+              :referer         => nil) 
+    }
 
-  it "should not match a nonsense line" do
-    @file_format.parse_line('dsadasdas dsaadsads dsaadsads').should be_nil
+    it { should_not parse_line('nonsense', 'a nonsense line') }
   end
-
-  it "should parse and convert the sample fields correctly" do
-    @log_parser.parse_io(StringIO.new(@sample)) do |request|
-      request[:bucket_owner].should    == '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1'
-      request[:bucket].should          == 'sample-bucket'
-      request[:remote_ip].should       == '207.171.172.6'
-      request[:key].should             == 'object.png'
-      request[:operation].should       == 'REST.GET.OBJECT'
-      request[:requester].should       == '65a011a29cdf8ec533ec3d1ccaae921c'
-      request[:request_id].should      == 'C980091AD89C936A'
-      request[:request_uri].should     == 'GET /sample-bucket/object.png HTTP/1.1'
-      request[:error_code].should      == nil
-      request[:http_status].should     == 200
-      request[:total_time].should      == 0.988
-      request[:turnaround_time].should == 0.987
-      request[:bytes_sent].should      == 1243
-      request[:object_size].should     == 1243
-      request[:user_agent].should      == 'aranhabot'
-      request[:referer].should         == nil
+  
+  describe '#parse_io' do
+    let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
+    let(:snippet) { log_snippet(sample_get, sample_copy, 'nonsense line') }
+    
+    it "should parse requests correctly and not generate warnings" do
+      log_parser.should_receive(:handle_request).twice
+      log_parser.should_not_receive(:warn)
+      log_parser.parse_io(snippet)
     end
   end
-
-  it "should parse a COPY request correctly" do
-    line = '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -'
-    @file_format.should parse_line(line).as(:access).and_capture(
-        :bucket_owner    => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a', 
-        :bucket          => 'testbucket', 
-        :timestamp       => 20100303230459, 
-        :remote_ip       => '174.119.31.76', 
-        :requester       => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
-        :key             => 'files/image.png', 
-        :operation       => 'REST.COPY.OBJECT', 
-        :total_time      => 0.365, 
-        :turnaround_time => 0.319, 
-        :bytes_sent      => 234, 
-        :object_size     => 65957, 
-        :referer         => nil,
-        :user_agent      => '')
-  end
-
 end