request-log-analyzer 1.9.10 → 1.11.0

data/lib/request_log_analyzer/file_format.rb

@@ -13,6 +13,8 @@ module RequestLogAnalyzer::FileFormat
   autoload :DelayedJob21,     'request_log_analyzer/file_format/delayed_job21'
   autoload :Apache,           'request_log_analyzer/file_format/apache'
   autoload :AmazonS3,         'request_log_analyzer/file_format/amazon_s3'
+  autoload :W3c,              'request_log_analyzer/file_format/w3c'
+  autoload :Haproxy,          'request_log_analyzer/file_format/haproxy'
 
   # Loads a FileFormat::Base subclass instance.
   # You can provide:
@@ -32,7 +34,7 @@ module RequestLogAnalyzer::FileFormat
 
     elsif file_format.kind_of?(String) && File.exist?(file_format) && File.file?(file_format)
       # load a format from a ruby file
-      require file_format
+      require File.expand_path(file_format)
 
       const = RequestLogAnalyzer.to_camelcase(File.basename(file_format, '.rb'))
       if RequestLogAnalyzer::FileFormat.const_defined?(const)
@@ -119,6 +121,18 @@ module RequestLogAnalyzer::FileFormat
       'Z' => '(?:[+-]\d{4}|[A-Z]{3,4})',
       '%' => '%'
     }
+    
+    # Creates a regular expression to match a hostname
+    def hostname(blank = false)
+      regexp = /(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])/
+      add_blank_option(regexp, blank)
+    end
+    
+    # Creates a regular expression to match a hostname or ip address
+    def hostname_or_ip_address(blank = false)
+      regexp = Regexp.union(hostname, ip_address)
+      add_blank_option(regexp, blank)
+    end
 
     # Create a regular expression for a timestamp, generated by a strftime call.
     # Provide the format string to construct a matching regular expression.
@@ -137,12 +151,7 @@ module RequestLogAnalyzer::FileFormat
         end
       end
 
-      regexp = Regexp.new(regexp)
-      return case blank
-        when String then Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
-        when true   then Regexp.union(regexp, //)
-        else regexp
-      end
+      add_blank_option(Regexp.new(regexp), blank)
     end
 
     # Construct a regular expression to parse IPv4 and IPv6 addresses.
@@ -160,12 +169,22 @@ module RequestLogAnalyzer::FileFormat
       ipv6_regex_compressed_hex_4_dec = /(?:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::(?:(?:[0-9A-Fa-f]{1,4}:)*)#{ipv4_regexp}/
       ipv6_regexp                     = Regexp.union(ipv6_regex_8_hex, ipv6_regex_compressed_hex, ipv6_regex_6_hex_4_dec, ipv6_regex_compressed_hex_4_dec)
 
-      # Allow the field to be blank if this option is given. This can be true to 
-      # allow an empty string or a string alternative for the nil value.
-      ip_regexp = case blank
-        when String then Regexp.union(ipv4_regexp, ipv6_regexp, Regexp.new(Regexp.quote(blank)))
-        when true   then Regexp.union(ipv4_regexp, ipv6_regexp, //)
-        else             Regexp.union(ipv4_regexp, ipv6_regexp)
+      add_blank_option(Regexp.union(ipv4_regexp, ipv6_regexp), blank)
+    end
+    
+    def anchored(regexp)
+      /^#{regexp}$/
+    end
+    
+    protected
+    
+    # Allow the field to be blank if this option is given. This can be true to 
+    # allow an empty string or a string alternative for the nil value.
+    def add_blank_option(regexp, blank)
+      case blank
+        when String; Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
+        when true;   Regexp.union(regexp, //)
+        else regexp
       end
     end
   end
@@ -258,9 +277,12 @@ module RequestLogAnalyzer::FileFormat
     end
 
     # Checks whether the file format is valid so it can be safely used with RLA.
-    def valid?
+    def well_formed?
       valid_line_definitions? && valid_request_class?
     end
+    
+    alias_method :valid?, :well_formed?
+  
 
     # Checks whether the line definitions form a valid language.
     # A file format should have at least a header and a footer line type    
@@ -292,5 +314,14 @@ module RequestLogAnalyzer::FileFormat
 
       return nil
     end
+    
+    # Returns the max line length for this file format if any.
+    def max_line_length
+      self.class.const_get(MAX_LINE_LENGTH) if self.class.const_defined?(:MAX_LINE_LENGTH)
+    end
+    
+    def line_divider
+      self.class.const_get(LINE_DIVIDER) if self.class.const_defined?(:LINE_DIVIDER)
+    end
   end
-end
+end

data/lib/request_log_analyzer/filter.rb

@@ -1,9 +1,5 @@
 module RequestLogAnalyzer::Filter
 
-  autoload :Field,     'request_log_analyzer/filter/field'
-  autoload :Timespan,  'request_log_analyzer/filter/timespan'
-  autoload :Anonymize, 'request_log_analyzer/filter/anonymize'
-
   # Base filter class used to filter input requests.
   # All filters should interit from this base.
   class Base
@@ -24,5 +20,8 @@ module RequestLogAnalyzer::Filter
       request
     end
   end
+end
 
-end
+require 'request_log_analyzer/filter/field'
+require 'request_log_analyzer/filter/timespan'
+require 'request_log_analyzer/filter/anonymize'

data/lib/request_log_analyzer/line_definition.rb

@@ -70,6 +70,10 @@ module RequestLogAnalyzer
       captures << new_capture_hash
       CaptureDefiner.new(new_capture_hash)
     end
+    
+    def all_captured_variables
+      captures.map { |c| c[:name] } + captures.map { |c| c[:provides] }.compact.map { |pr| pr.keys }.flatten
+    end
 
     # Checks whether a given line matches this definition.
     # It will return false if a line does not match. If the line matches, a hash is returned
@@ -128,9 +132,7 @@ module RequestLogAnalyzer
 
     # Returns true if this line captures values of the given name
     def captures?(name)
-      captures.any? { |c| c[:name] == name }
+      all_captured_variables.include?(name)
     end
-
   end
-
-end
+end

data/lib/request_log_analyzer/output.rb

@@ -1,10 +1,6 @@
 # Module for generating output
 module RequestLogAnalyzer::Output
 
-  autoload :FixedWidth, 'request_log_analyzer/output/fixed_width'
-  autoload :HTML,       'request_log_analyzer/output/html'
-  autoload :FancyHTML,  'request_log_analyzer/output/fancy_html'
-  
   # Loads a Output::Base subclass instance.
   def self.load(file_format, *args)
     
@@ -111,6 +107,8 @@ module RequestLogAnalyzer::Output
     def table_has_header?(columns)
       columns.any? { |column| !column[:title].nil? }
     end
-
   end
 end
+
+require 'request_log_analyzer/output/fixed_width'
+require 'request_log_analyzer/output/html'

data/lib/request_log_analyzer/request.rb

@@ -27,6 +27,13 @@ module RequestLogAnalyzer
       def convert_integer(value, capture_definition); value.to_i; end
       def convert_sym(value, capture_definition);     value.to_sym; end
       def convert_symbol(value, capture_definition);  value.to_sym; end
+      def convert_nillable_string(value, definition); value == '-' ? nil : value ; end
+      
+      # This function can be overridden to rewrite the path for better categorization in the
+      # reports.
+      def convert_path(value, definition)
+        value
+      end
 
       # Converts :eval field, which should evaluate to a hash.
       def convert_eval(value, capture_definition)

data/lib/request_log_analyzer/source/log_parser.rb

@@ -13,6 +13,11 @@ module RequestLogAnalyzer::Source
 
     include Enumerable
 
+    # The maximum number of bytes to read from a line.
+    DEFAULT_MAX_LINE_LENGTH = 8096
+    
+    DEFAULT_LINE_DIVIDER = "\n"
+
     # The default parse strategy that will be used to parse the input.
     DEFAULT_PARSE_STRATEGY = 'assume-correct'
 
@@ -48,6 +53,14 @@ module RequestLogAnalyzer::Source
       end
     end
 
+    def max_line_length
+      file_format.max_line_length || DEFAULT_MAX_LINE_LENGTH
+    end
+    
+    def line_divider
+      file_format.line_divider || DEFAULT_LINE_DIVIDER
+    end
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
@@ -132,6 +145,13 @@ module RequestLogAnalyzer::Source
     def parse_stream(stream, options = {}, &block)
       parse_io(stream, options, &block)
     end
+    
+    # Parses a string. It will simply call parse_io. This function does not support progress updates.
+    # <tt>string</tt>:: The string that should be parsed.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def parse_string(string, options = {}, &block)
+      parse_io(StringIO.new(string), options, &block)
+    end
 
     # This method loops over each line of the input stream. It will try to parse this line as any of
     # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
@@ -143,11 +163,15 @@ module RequestLogAnalyzer::Source
     # - The method will yield progress updates if a progress handler is installed using progress=
     # - The method will yield parse warnings if a warning handler is installed using warning=
     #
+    # This is a Ruby 1.9 specific version that offers memory protection.
+    #
     # <tt>io</tt>:: The IO instance to use as source
     # <tt>options</tt>:: A hash of options that can be used by the parser.
-    def parse_io(io, options = {}, &block) # :yields: request
-      @current_lineno = 0
-      while line = io.gets
+    def parse_io_19(io, options = {}, &block) # :yields: request
+      @max_line_length = options[:max_line_length] || max_line_length
+      @line_divider    = options[:line_divider]    || line_divider
+      @current_lineno  = 0
+      while line = io.gets(@line_divider, @max_line_length)
         @current_lineno += 1
         @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
         parse_line(line, &block)
@@ -157,6 +181,35 @@ module RequestLogAnalyzer::Source
       @current_lineno = nil
     end
     
+    # This method loops over each line of the input stream. It will try to parse this line as any of
+    # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
+    # It will then combine these parsed line into requests using heuristics. These requests (see
+    # RequestLogAnalyzer::Request) will then be yielded for further processing in the pipeline.
+    #
+    # - RequestLogAnalyzer::LineDefinition#matches is called to test if a line matches a line definition of the file format.
+    # - update_current_request is used to combine parsed lines into requests using heuristics.
+    # - The method will yield progress updates if a progress handler is installed using progress=
+    # - The method will yield parse warnings if a warning handler is installed using warning=
+    #
+    # This is a Ruby 1.8 specific version that doesn't offer memory protection.
+    #
+    # <tt>io</tt>:: The IO instance to use as source
+    # <tt>options</tt>:: A hash of options that can be used by the parser.    
+    def parse_io_18(io, options = {}, &block) # :yields: request
+      @line_divider    = options[:line_divider]    || line_divider
+      @current_lineno  = 0
+      while line = io.gets(@line_divider)
+        @current_lineno += 1
+        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
+        parse_line(line, &block)
+      end
+
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+      @current_lineno = nil
+    end
+    
+    alias_method :parse_io, RUBY_VERSION.to_f < 1.9 ? :parse_io_18 : :parse_io_19
+    
     # Parses a single line using the current file format. If successful, use the parsed
     # information to build a request
     # <tt>line</tt>:: The line to parse
@@ -295,5 +348,4 @@ module RequestLogAnalyzer::Source
       hash[:line_definition].footer
     end
   end
-
 end

data/lib/request_log_analyzer/source.rb

@@ -8,9 +8,6 @@
 # - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
 module RequestLogAnalyzer::Source
 
-  autoload :LogParser,      'request_log_analyzer/source/log_parser'
-  autoload :DatabaseLoader, 'request_log_analyzer/source/database_loader'
-
   # The base Source class. All other sources should inherit from this class.
   #
   # A source implememtation should at least implement the each_request method, which should yield
@@ -65,4 +62,6 @@ module RequestLogAnalyzer::Source
     end
 
   end
-end
+end
+
+require 'request_log_analyzer/source/log_parser'

data/lib/request_log_analyzer/tracker.rb

@@ -1,12 +1,5 @@
 module RequestLogAnalyzer::Tracker
 
-  autoload :Duration,     'request_log_analyzer/tracker/duration'
-  autoload :Frequency,    'request_log_analyzer/tracker/frequency'
-  autoload :HourlySpread, 'request_log_analyzer/tracker/hourly_spread'
-  autoload :NumericValue, 'request_log_analyzer/tracker/numeric_value'
-  autoload :Timespan,     'request_log_analyzer/tracker/timespan'
-  autoload :Traffic,      'request_log_analyzer/tracker/traffic'
-
   # Base Tracker class. All other trackers inherit from this class
   #
   # Accepts the following options:
@@ -99,4 +92,11 @@ module RequestLogAnalyzer::Tracker
       nil
     end
   end
-end
+end
+
+require 'request_log_analyzer/tracker/numeric_value'
+require 'request_log_analyzer/tracker/duration'
+require 'request_log_analyzer/tracker/frequency'
+require 'request_log_analyzer/tracker/hourly_spread'
+require 'request_log_analyzer/tracker/timespan'
+require 'request_log_analyzer/tracker/traffic'

data/lib/request_log_analyzer.rb

@@ -10,36 +10,10 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
 # The {RequestLogAnalyzer::VERSION} constant can be used to determine what version of request-log-analyzer
 # is running.
 module RequestLogAnalyzer
-
+  
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.9.10"
-
-
-  autoload :Controller,     'request_log_analyzer/controller'
-  autoload :Aggregator,     'request_log_analyzer/aggregator'
-  autoload :Database,       'request_log_analyzer/database'
-  autoload :FileFormat,     'request_log_analyzer/file_format'
-  autoload :Filter,         'request_log_analyzer/filter'
-  autoload :LineDefinition, 'request_log_analyzer/line_definition'
-  autoload :LogProcessor,   'request_log_analyzer/log_processor'
-  autoload :Mailer,         'request_log_analyzer/mailer'
-  autoload :Output,         'request_log_analyzer/output'
-  autoload :Request,        'request_log_analyzer/request'
-  autoload :Source,         'request_log_analyzer/source'
-  autoload :Tracker,        'request_log_analyzer/tracker'
-
-  # Loads constants that reside in the RequestLogAnalyzer tree using the constant name and its base 
-  # constant to determine the filename.
-  # @param [Module] base The base constant to load the constant from. This should be <tt>Foo</tt> when
-  #        the constant <tt>Foo::Bar</tt> is being loaded.
-  # @param [Symbol] const The constant to load from the base constant as a string or symbol. This 
-  #        should be <tt>"Bar"<tt> or <tt>:Bar</tt> when the constant <tt>Foo::Bar</tt> is being loaded.
-  # @return [Module] The loaded module, nil if it was not found on the expected location.
-  def self.load_default_class_file(base, const)
-    require "#{to_underscore("#{base.name}::#{const}")}"
-    base.const_get(const) if base.const_defined?(const)
-  end
+  VERSION = "1.11.0"
 
   # Convert a string/symbol in camelcase ({RequestLogAnalyzer::Controller}) to underscores 
   # (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using 
@@ -55,9 +29,21 @@ module RequestLogAnalyzer
   # ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given 
   # filename.
   #
-  # @param [#to_s] str The string-like to convert in the following format: <tt>module_name/class_name</tt>.
+  # @param [#to_s] str The string-like to convert in the f`ollowing format: <tt>module_name/class_name</tt>.
   # @return [String] The input string converted to camelcase form.  
   def self.to_camelcase(str)
     str.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
   end
 end
+
+require 'request_log_analyzer/controller'
+require 'request_log_analyzer/aggregator'
+require 'request_log_analyzer/file_format'
+require 'request_log_analyzer/filter'
+require 'request_log_analyzer/line_definition'
+require 'request_log_analyzer/log_processor'
+require 'request_log_analyzer/mailer'
+require 'request_log_analyzer/output'
+require 'request_log_analyzer/request'
+require 'request_log_analyzer/source'
+require 'request_log_analyzer/tracker'

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.9.10"
-  s.date    = "2011-01-03"
+  s.version = "1.11.0"
+  s.date    = "2011-04-27"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -39,6 +39,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(.gitignore DESIGN.rdoc LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/mixins/gets_memory_protection.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fancy_html.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
-  s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
+  s.files      = %w(.gitignore .infinity_test DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
+  s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
 end

data/spec/fixtures/mysql_slow_query.log

@@ -1,4 +1,3 @@
-SELECT /*!40001 SQL_NO_CACHE */ * FROM `clients`;
 # Time: 091112 18:13:56
 # User@Host: admin[admin] @ db1 [10.0.0.1]
 # Query_time: 10  Lock_time: 0  Rows_sent: 1191307  Rows_examined: 1191307

data/spec/integration/command_line_usage_spec.rb

@@ -14,11 +14,6 @@ describe RequestLogAnalyzer, 'running from command line' do
     output = run("#{log_fixture(:rails_1x)}")
     output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
   end
-
-  it "should function correctly with the gets mixin" do
-    output = run("#{log_fixture(:rails_1x)} --gets-memory-protection")
-    output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
-  end
   
   it "should find 2 requests when parsing a compressed file" do
     output = run("#{log_fixture(:decompression, :tgz)}")

data/spec/lib/helpers.rb

@@ -11,8 +11,8 @@ module RequestLogAnalyzer::RSpec::Helpers
   end
 
   # Creates a log file given some lines
-  def log_stream(*lines)
-    StringIO.new(lines.join("\n") + "\n")
+  def log_snippet(*lines)
+    StringIO.new(lines.join("\n") << "\n")
   end
 
   # Request loopback

data/spec/lib/matchers.rb

@@ -7,15 +7,23 @@ module RequestLogAnalyzer::RSpec::Matchers
       @captures  = []
     end
 
-    def capturing(*captures)
+    def and_capture(*captures)
       @captures += captures
       return self
     end
+    
+    alias_method :capturing, :and_capture
+    
+    def description
+      description = "have a #{@line_type.inspect} line definition"
+      description << " that captures #{@captures.join(', ')}" unless @captures.empty?
+      description
+    end
 
     def matches?(file_format)
       file_format = file_format.create if file_format.kind_of?(Class)
       if ld = file_format.line_definitions[@line_type]
-        @captures.all? { |c| ld.captures.map { |cd| cd[:name] }.include?(c) }
+        @captures.all? { |c| ld.all_captured_variables.include?(c) }
       else
         false
       end
@@ -24,10 +32,25 @@ module RequestLogAnalyzer::RSpec::Matchers
 
   class ParseLine
     
-    def initialize(line)
+    def initialize(line, line_description = nil)
       @line      = line
       @captures  = {}
       @line_type = nil
+      @line_description = line_description
+    end
+
+    def line_description
+      @full_line_description ||= if @line_description
+        if @line_type && @line_description =~ /^(?:with|without|having|using) /
+          "a #{@line_type.inspect} line #{@line_description}"
+        else
+          @line_description
+        end
+      elsif @line_type 
+        "a #{@line_type.inspect} line"
+      else
+        "line #{@line.inspect}"
+      end
     end
 
     def failure_message
@@ -43,11 +66,20 @@ module RequestLogAnalyzer::RSpec::Matchers
       @captures = captures
       return self
     end
+    
+    alias_method :capturing, :and_capture
 
     def fail(message)
       @failure_message = message
       return false
     end
+    
+    def description
+      description = "parse #{line_description}"
+      description << " as line type #{@line_type.inspect}" if @line_type
+      description << " and capture #{@captures.keys.join(', ')} correctly" unless @captures.empty?
+      description
+    end
 
     def matches?(file_format)
       if @line_hash = file_format.parse_line(@line)
@@ -70,8 +102,7 @@ module RequestLogAnalyzer::RSpec::Matchers
     return HasLineDefinition.new(line_type)
   end
 
-  def parse_line(line)
-    ParseLine.new(line)
+  def parse_line(line, line_description = nil)
+    ParseLine.new(line, line_description)
   end
-
-end
+end

data/spec/lib/mocks.rb

@@ -70,8 +70,4 @@ module RequestLogAnalyzer::RSpec::Mocks
     connection.stub!(:table_creator).and_return(table_creator)
     return connection
   end
-
-  def request_counter
-    @request_counter ||= mock('aggregator to count request')
-  end
-end
+end

data/spec/spec_helper.rb

@@ -1,8 +1,9 @@
-$LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
-
 require 'rubygems'
+require 'bundler/setup'
+
 require 'rspec'
 require 'request_log_analyzer'
+require 'stringio'
 
 module RequestLogAnalyzer::RSpec
 end

data/spec/unit/database/base_class_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'request_log_analyzer/database'
 
 describe RequestLogAnalyzer::Database::Base do