request-log-analyzer 1.9.10 → 1.10.0

data/.gitignore

@@ -8,4 +8,6 @@ requests.db
 /tmp
 /classes
 /files
-/coverage
+/coverage
+Gemfile.lock
+/.bundle/

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/lib/request_log_analyzer.rb

@@ -13,7 +13,7 @@ module RequestLogAnalyzer
 
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.9.10"
+  VERSION = "1.10.0"
 
 
   autoload :Controller,     'request_log_analyzer/controller'

data/lib/request_log_analyzer/file_format.rb

@@ -13,6 +13,8 @@ module RequestLogAnalyzer::FileFormat
   autoload :DelayedJob21,     'request_log_analyzer/file_format/delayed_job21'
   autoload :Apache,           'request_log_analyzer/file_format/apache'
   autoload :AmazonS3,         'request_log_analyzer/file_format/amazon_s3'
+  autoload :W3c,              'request_log_analyzer/file_format/w3c'
+  autoload :Haproxy,          'request_log_analyzer/file_format/haproxy'
 
   # Loads a FileFormat::Base subclass instance.
   # You can provide:
@@ -293,4 +295,4 @@ module RequestLogAnalyzer::FileFormat
       return nil
     end
   end
-end
+end

data/lib/request_log_analyzer/file_format/apache.rb

@@ -30,20 +30,23 @@ module RequestLogAnalyzer::FileFormat
 
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
-      '%' => { :regexp => '%', :captures => [] },
-      'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
-      'a' => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] },
-      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
-      'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
-      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
-      'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
-      'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
-      't' => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] },
-      's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
-      'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
-      'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
-      'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
-                       {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
+      '%' => { nil => { :regexp => '%', :captures => [] } },
+      'h' => { nil => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] } },
+      'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
+      'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
+      'c' => { nil => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] } },
+      'D' => { nil     => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
+               'micro' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
+               'milli' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :msec }] }
+             },
+      'l' => { nil => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] } },
+      'T' => { nil => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] } },
+      't' => { nil => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] } },
+      's' => { nil => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] } },
+      'u' => { nil => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] } },
+      'U' => { nil => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] } },
+      'r' => { nil => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
+                       {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]} },
       'i' => { 'Referer'    => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
                'User-agent' => { :regexp => '(.*)',  :captures => [{:name => :user_agent, :type => :user_agent}] }
              }
@@ -72,14 +75,13 @@ module RequestLogAnalyzer::FileFormat
 
         if variable
           # Check if we recognize the log directive
-          directive = LOG_DIRECTIVES[variable]
-          directive = directive[arg] if directive && arg
+          directive = LOG_DIRECTIVES[variable][arg] rescue nil
 
           if directive
             line_regexp << directive[:regexp]   # Parse the value of the directive
             captures    += directive[:captures] # Add the directive's information to the captures
           else
-            puts "%#{directive} log directiven not yet supported, field is ignored."
+            puts "Apache log directive %#{arg}#{variable} is not yet supported by RLA, the field will be ignored."
             line_regexp << '.*' # Just accept any input for this literal
           end
         end
@@ -126,22 +128,11 @@ module RequestLogAnalyzer::FileFormat
         "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
 
-      # This function can be overridden to rewrite the path for better categorization in the
-      # reports.
-      def convert_path(value, definition)
-        value
-      end
-
       # This function can be overridden to simplify the user agent string for better
       # categorization in the reports
       def convert_user_agent(value, definition)
         value # TODO
       end
-
-      # Make sure that the string '-' is parsed as a nil value.
-      def convert_nillable_string(value, definition)
-        value == '-' ? nil : value
-      end
     end
   end
 end

data/lib/request_log_analyzer/file_format/haproxy.rb

@@ -0,0 +1,138 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Haproxy < RequestLogAnalyzer::FileFormat::Base
+
+    extend CommonRegularExpressions
+
+    # Define line types
+    line_definition :haproxy do |line|
+      line.header = true
+      line.footer = true
+
+      line.regexp = %r{
+        (#{ip_address}):\d+\s # client_ip ':' client_port
+        \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})\.\d{3}\]\s # '[' accept_date ']'
+        (\S+)\s # frontend_name
+        (\S+)\/(\S+)\s # backend_name '/' server_name
+        (\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        (\d+)\s # status_code
+        \+?(\d+)\s # bytes_read
+        (\S+)\s # captured_request_cookie
+        (\S+)\s # captured_response_cookie
+        (\w|-)(\w|-)(\w|-)(\w|-)\s # termination_state
+        (\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
+        (\d+)\/(\d+)\s # srv_queue '/' backend_queue
+        (\S*)\s? # captured_request_headers
+        (\S*)\s? # captured_response_headers
+        "([^"]*)" # '"' http_request '"'
+      }x
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:accept_date).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:backend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tw).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:actconn).as(:integer)
+      line.capture(:feconn).as(:integer)
+      line.capture(:beconn).as(:integer)
+      line.capture(:srv_conn).as(:integer)
+      line.capture(:retries).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
+    # Define the summary report
+    report do |analyze|
+      analyze.hourly_spread :field => :accept_date
+
+      analyze.frequency :client_ip,
+        :title => "Hits per IP"
+
+      analyze.frequency :frontend_name,
+        :title => "Hits per frontend service"
+
+      analyze.frequency :backend_name,
+        :title => "Hits per backend service"
+
+      analyze.frequency :server_name,
+        :title => "Hits per backend server"
+
+      analyze.frequency :status_code,
+        :title => "HTTP response code frequency"
+
+      analyze.frequency :http_request,
+        :title => "Most popular requests"
+
+      analyze.frequency :http_request,
+        :title => "Most frequent HTTP 40x errors",
+        :category => lambda { |r| "#{r[:http_request]}"},
+        :if => lambda { |r| r[:status_code] >= 400 and r[:status_code] <= 417 }
+
+      analyze.frequency :http_request,
+        :title => "Most frequent HTTP 50x errors",
+        :category => lambda { |r| "#{r[:http_request]}"},
+        :if => lambda { |r| r[:status_code] >= 500 and r[:status_code] <= 505 }
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per frontend service",
+        :category => lambda { |r| "#{r[:frontend_name]}"}
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per backend service",
+        :category => lambda { |r| "#{r[:backend_name]}"}
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per backend server",
+        :category => lambda { |r| "#{r[:server_name]}"}
+
+      analyze.duration :tr,
+        :title => "Time waiting for backend response",
+        :category => lambda { |r| "#{r[:http_request]}"}
+
+      analyze.duration :tt,
+        :title => "Total time spent on request",
+        :category => lambda { |r| "#{r[:http_request]}"}
+    end
+
+    # Define a custom Request class for the HAProxy file format to speed up
+    # timestamp handling. Shamelessly copied from apache.rb
+    class Request < RequestLogAnalyzer::Request
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return
+      # a integer to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
+      end
+
+      # Make sure that the strings '-' or '{}' or '' are parsed as a nil value.
+      def convert_nillable_string(value, definition)
+        value =~ /-|\{\}|^$/ ? nil : value
+      end
+
+      # Make sure that -1 is parsed as a nil value.
+      def convert_nillable_duration(value, definition)
+        value == '-1' ? nil : convert_duration(value, definition)
+      end
+
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -36,7 +36,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :completed do |line|
       line.footer = true
       line.teaser = /Completed /
-      line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms\))?/
+      line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms.*\))?/
       
       line.capture(:status).as(:integer)
       line.capture(:duration).as(:duration, :unit => :msec)

data/lib/request_log_analyzer/file_format/w3c.rb

@@ -0,0 +1,56 @@
+module RequestLogAnalyzer::FileFormat
+
+  # FileFormat for W3C access logs.
+  class W3c < Base
+    
+    extend CommonRegularExpressions
+
+    line_definition :access do |line|
+      line.header = true
+      line.footer = true
+      line.regexp = /^(#{timestamp('%Y-%m-%d %H:%M:%S')}) (#{ip_address}) (.*) (#{ip_address}) (\d+) (\w+) ([\w|\/|.]+) \- (\d+) (\d+) (\d+) (\d+) (.*) (\S+)/
+      
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:remote_ip)
+      line.capture(:username).as(:nillable_string)
+      line.capture(:local_ip)
+      line.capture(:port).as(:integer)
+      line.capture(:method)
+      line.capture(:path).as(:path)
+      line.capture(:http_status).as(:integer)
+      line.capture(:bytes_sent).as(:traffic, :unit => :byte)
+      line.capture(:bytes_received).as(:traffic, :unit => :byte)
+      line.capture(:duration).as(:duration, :unit => :msec)
+      line.capture(:user_agent)
+      line.capture(:referer)
+    end
+
+    report do |analyze|
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :category => :http_method, :title => "HTTP methods"
+      analyze.frequency :category => :http_status, :title => "HTTP statuses"
+      
+      analyze.frequency :category => :path, :title => "Most popular URIs"
+
+      analyze.frequency :category => :user_agent, :title => "User agents"
+      analyze.frequency :category => :referer,    :title => "Referers"
+
+      analyze.duration :duration => :duration,  :category => :path, :title => 'Request duration'
+      analyze.traffic  :traffic => :bytes_sent, :category => :path, :title => 'Traffic out'
+      analyze.traffic  :traffic => :bytes_received, :category => :path, :title => 'Traffic in'
+    end
+    
+    class Request < RequestLogAnalyzer::Request
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
+      # to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[0,4]}#{value[5,2]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
+      end
+    end
+    
+
+  end
+  
+end

data/lib/request_log_analyzer/request.rb

@@ -27,6 +27,13 @@ module RequestLogAnalyzer
       def convert_integer(value, capture_definition); value.to_i; end
       def convert_sym(value, capture_definition);     value.to_sym; end
       def convert_symbol(value, capture_definition);  value.to_sym; end
+      def convert_nillable_string(value, definition); value == '-' ? nil : value ; end
+      
+      # This function can be overridden to rewrite the path for better categorization in the
+      # reports.
+      def convert_path(value, definition)
+        value
+      end
 
       # Converts :eval field, which should evaluate to a hash.
       def convert_eval(value, capture_definition)

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.9.10"
-  s.date    = "2011-01-03"
+  s.version = "1.10.0"
+  s.date    = "2011-02-26"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -39,6 +39,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(.gitignore DESIGN.rdoc LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/mixins/gets_memory_protection.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fancy_html.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
-  s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
+  s.files      = %w(.gitignore DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/mixins/gets_memory_protection.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fancy_html.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
+  s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
 end

data/spec/spec_helper.rb

@@ -1,8 +1,9 @@
-$LOAD_PATH << File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
-
 require 'rubygems'
+require 'bundler/setup'
+
 require 'rspec'
 require 'request_log_analyzer'
+require 'stringio'
 
 module RequestLogAnalyzer::RSpec
 end

data/spec/unit/file_format/apache_format_spec.rb

@@ -28,6 +28,23 @@ describe RequestLogAnalyzer::FileFormat::Apache do
       @line_definition.captures?(:duration).should be_true
     end
   end
+  
+  describe '.access_line_definition' do
+    it "should parse values in microseconds when no argument is given to %D" do
+      format = RequestLogAnalyzer::FileFormat::Apache.create('%D')
+      format.should parse_line('12345').and_capture(:duration => 0.012345)
+    end
+    
+    it "should parse values in microseconds when micro is given as argument to %D" do
+      format = RequestLogAnalyzer::FileFormat::Apache.create('%{micro}D')
+      format.should parse_line('12345').and_capture(:duration => 0.012345)
+    end
+    
+    it "should parse values in microseconds when micro is given as argument to %D" do
+      format = RequestLogAnalyzer::FileFormat::Apache.create('%{milli}D')
+      format.should parse_line('12345').and_capture(:duration => 12.345)
+    end
+  end
 
   describe '.create' do
 

data/spec/unit/file_format/haproxy_format_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Haproxy do
+
+  before(:each) do
+    @file_format = RequestLogAnalyzer::FileFormat.load(:haproxy)
+    @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+    @sample1 = 'Feb  6 12:14:14 localhost haproxy[14389]: 10.0.1.2:33317 [06/Feb/2009:12:14:14.655] http-in static/srv1 10/0/30/69/109 200 2750 - - ---- 1/1/1/1/0 0/0 {1wt.eu} {} "GET /index.html HTTP/1.1"'
+    @sample2 = 'haproxy[18113]: 127.0.0.1:34549 [15/Oct/2003:15:19:06.103] px-http px-http/<NOSRV> -1/-1/-1/-1/+50001 408 +2750 - - cR-- 2/2/2/0/+2 0/0 ""'
+  end
+
+  it "should be a valid file format" do
+    @file_format.should be_valid
+  end
+
+  it "should parse access lines and capture all of its fields" do
+    @file_format.should have_line_definition(:haproxy).capturing(:client_ip, :accept_date, :frontend_name, :backend_name, :server_name, :tq, :tw, :tc, :tr, :tt, :status_code, :bytes_read, :captured_request_cookie, :captured_response_cookie, :termination_event_code, :terminated_session_state, :clientside_persistence_cookie, :serverside_persistence_cookie, :actconn, :feconn, :beconn, :srv_conn, :retries, :srv_queue, :backend_queue, :captured_request_headers, :captured_response_headers, :http_request)
+  end
+
+  it "should match the sample line" do
+    @file_format.parse_line(@sample1).should include(:line_definition, :captures)
+  end
+
+  it "should not match a nonsense line" do
+    @file_format.parse_line('dsadasdas dsaadsads dsaadsads').should be_nil
+  end
+
+  it "should parse and convert the sample fields correctly" do
+    @log_parser.parse_io(StringIO.new(@sample1)) do |request|
+      request[:client_ip].should                      == '10.0.1.2'
+      request[:accept_date].should                    == 20090206121414
+      request[:frontend_name].should                  == 'http-in'
+      request[:backend_name].should                   == 'static'
+      request[:server_name].should                    == 'srv1'
+      request[:tq].should                             == 0.010
+      request[:tw].should                             == 0.000
+      request[:tc].should                             == 0.030
+      request[:tr].should                             == 0.069
+      request[:tt].should                             == 0.109
+      request[:status_code].should                    == 200
+      request[:bytes_read].should                     == 2750
+      request[:captured_request_cookie].should        == nil
+      request[:captured_response_cookie].should       == nil
+      request[:termination_event_code].should         == nil
+      request[:terminated_session_state].should       == nil
+      request[:clientside_persistence_cookie].should  == nil
+      request[:serverside_persistence_cookie].should  == nil
+      request[:actconn].should                        == 1
+      request[:feconn].should                         == 1
+      request[:beconn].should                         == 1
+      request[:srv_conn].should                       == 1
+      request[:retries].should                        == 0
+      request[:srv_queue].should                      == 0
+      request[:backend_queue].should                  == 0
+      request[:captured_request_headers].should       == '{1wt.eu}'
+      request[:captured_response_headers].should      == nil
+      request[:http_request].should                   == 'GET /index.html HTTP/1.1'
+    end
+  end
+
+  it "should parse and convert edge case sample fields correctly" do
+    @log_parser.parse_io(StringIO.new(@sample2)) do |request|
+      request[:accept_date].should                    == 20031015151906
+      request[:server_name].should                    == '<NOSRV>'
+      request[:tq].should                             == nil
+      request[:tw].should                             == nil
+      request[:tc].should                             == nil
+      request[:tr].should                             == nil
+      request[:tt].should                             == 50.001
+      request[:bytes_read].should                     == 2750
+      request[:captured_request_cookie].should        == nil
+      request[:captured_response_cookie].should       == nil
+      request[:termination_event_code].should         == 'c'
+      request[:terminated_session_state].should       == 'R'
+      request[:clientside_persistence_cookie].should  == nil
+      request[:serverside_persistence_cookie].should  == nil
+      request[:retries].should                        == 2
+      request[:captured_request_headers].should       == nil
+      request[:captured_response_headers].should      == nil
+      request[:http_request].should                   == nil
+    end
+  end
+
+end

data/spec/unit/file_format/postgresql_format_spec.rb

@@ -2,36 +2,29 @@ require 'spec_helper'
 
 describe RequestLogAnalyzer::FileFormat::Postgresql do
 
-  it "should be a valid file format" do
-    RequestLogAnalyzer::FileFormat.load(:Postgresql).should be_valid
-  end
+  subject { RequestLogAnalyzer::FileFormat.load(:Postgresql) }
+  let(:log_parser)  { RequestLogAnalyzer::Source::LogParser.new(subject) }
+  
+  it { should be_valid }
 
   describe '#parse_line' do
-    before(:each) do
-      @file_format = RequestLogAnalyzer::FileFormat.load(:Postgresql)
-    end
-
     it "should parse a :query line correctly" do
       line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG:  00000: duration: 0.710 ms  statement: SELECT * FROM "delayed_jobs"'
-      @file_format.should parse_line(line).as(:query).and_capture(:timestamp => 20101010135207, :query_fragment => 'SELECT * FROM "delayed_jobs"')
+      subject.should parse_line(line).as(:query).and_capture(:timestamp => 20101010135207, :query_fragment => 'SELECT * FROM "delayed_jobs"')
     end
     
     it "should parse a :query_fragment line correctly" do
       line = '    ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",'
-      @file_format.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
+      subject.should parse_line(line).as(:query_fragment).and_capture(:query_fragment => '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",')
     end
 
     it "should parse a :query line correctly" do
       line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG:  00000: duration: 0.710 ms  statement: SELECT * FROM "delayed_jobs"'
-      @file_format.should parse_line(line).as(:query).and_capture(:query_time => 0.710)
-    end    
+      subject.should parse_line(line).as(:query).and_capture(:query_time => 0.710)
+    end
   end
 
   describe '#parse_io' do
-    before(:each) do
-      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:postgresql))
-    end
-
     it "should parse a multiline query entry correctly" do
       fixture = <<-EOS
         2010-10-10 15:00:02 GMT [38747]: [1669-1] LOG:  00000: duration: 0.195 ms  statement: INSERT INTO "delayed_jobs" ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at", "last_error") VALUES(NULL, NULL, '2010-10-10 15:00:02.159884', E'--- !ruby/object:RuntheChooChootrain {}
@@ -40,13 +33,11 @@ describe RequestLogAnalyzer::FileFormat::Postgresql do
       EOS
 
       request_counter.should_receive(:hit!).exactly(1).times
-      @log_parser.should_not_receive(:warn)
-      @log_parser.parse_io(StringIO.new(fixture)) do |request|
+      log_parser.should_not_receive(:warn)
+      log_parser.parse_io(StringIO.new(fixture)) do |request|
         request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Postgresql::Request) && request.completed?
         request[:query].should == 'INSERT INTO delayed_jobs (failed_at, locked_by, created_at, handler, updated_at, priority, run_at, attempts, locked_at, last_error) VALUES(NULL, NULL, :string, E:string, :string, :int, :string, :int, NULL, NULL) RETURNING id'
       end
     end
-
   end
 end
-

data/spec/unit/file_format/rails3_format_spec.rb

@@ -44,9 +44,9 @@ describe RequestLogAnalyzer::FileFormat::Rails do
     end
     
     it "should parse :completed lines correctly" do
-      line = 'Completed 200 OK in 170ms (Views: 78.4ms | ActiveRecord: 48.2ms)'
+      line = 'Completed 200 OK in 170ms (Views: 78.0ms | ActiveRecord: 48.2ms)'
       @file_format.should parse_line(line).as(:completed).and_capture(
-          :duration => 0.170, :status => 200)
+          :duration => 0.170, :view => 0.078, :db => 0.0482, :status => 200)
     end
     
     it "should parse :completed lines correctly when ActiveRecord is not mentioned" do
@@ -54,6 +54,12 @@ describe RequestLogAnalyzer::FileFormat::Rails do
       @file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.364, :status => 200)
     end
     
+    it "should parse :completed lines correctly when other durations are specified" do
+      line = 'Completed 200 OK in 384ms (Views: 222.0ms | ActiveRecord: 121.0ms | Sphinx: 0.0ms)'
+      @file_format.should parse_line(line).as(:completed).and_capture(:duration => 0.384, :view => 0.222, 
+          :db => 0.121, :status => 200)
+    end
+    
 
     it "should pase :failure lines correctly" do
       line = "ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:"

data/spec/unit/file_format/w3c_format_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::W3c do
+
+  before(:each) do
+    @file_format = RequestLogAnalyzer::FileFormat.load(:w3c)
+    @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)  
+    #          date       time     c-ip cs-username s-ip     s-port  cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs(User-Agent) cs(Referrer) 
+    @sample = '2002-05-24 20:18:01 172.224.24.114 - 206.73.118.24 80 GET /Default.htm - 200 7930 248 31 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server) http://64.224.24.114/'
+  end
+
+  it "should be a valid file format" do
+    @file_format.should be_valid
+  end
+
+  it "should parse access lines and capture all of its fields" do
+    @file_format.should have_line_definition(:access).capturing(:timestamp, :remote_ip, :username, :local_ip, :port, :method, :path, :http_status, :bytes_sent, :bytes_received, :duration, :user_agent, :referer)
+  end
+
+  it "should match the sample line" do
+    @file_format.parse_line(@sample).should include(:line_definition, :captures)
+  end
+
+  it "should not match a nonsense line" do
+    @file_format.parse_line('#Software: Microsoft Internet Information Services 6.0').should be_nil
+  end
+
+  it "should parse and convert the sample fields correctly" do
+    @log_parser.parse_io(StringIO.new(@sample)) do |request|
+      request[:timestamp].should      == 20020524201801
+      request[:remote_ip].should      == "172.224.24.114"
+      request[:username].should       == nil
+      request[:local_ip].should       == "206.73.118.24"
+      request[:port].should           == 80
+      request[:method].should         == 'GET'
+      request[:path].should           == '/Default.htm'
+      request[:http_status].should    == 200
+      request[:bytes_sent].should     == 7930
+      request[:bytes_received].should == 248
+      request[:duration].should       == 0.031
+      request[:user_agent].should     == 'Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server)'
+      request[:referer].should        == 'http://64.224.24.114/'
+    end
+  end
+
+end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 9
   - 10
-  version: 1.9.10
+  - 0
+  version: 1.10.0
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-03 00:00:00 +01:00
+date: 2011-02-26 00:00:00 -05:00
 default_executable: request-log-analyzer
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -84,6 +84,7 @@ extra_rdoc_files:
 files: 
 - .gitignore
 - DESIGN.rdoc
+- Gemfile
 - LICENSE
 - README.rdoc
 - Rakefile
@@ -112,6 +113,7 @@ files:
 - lib/request_log_analyzer/file_format/delayed_job.rb
 - lib/request_log_analyzer/file_format/delayed_job2.rb
 - lib/request_log_analyzer/file_format/delayed_job21.rb
+- lib/request_log_analyzer/file_format/haproxy.rb
 - lib/request_log_analyzer/file_format/merb.rb
 - lib/request_log_analyzer/file_format/mysql.rb
 - lib/request_log_analyzer/file_format/oink.rb
@@ -120,6 +122,7 @@ files:
 - lib/request_log_analyzer/file_format/rails.rb
 - lib/request_log_analyzer/file_format/rails3.rb
 - lib/request_log_analyzer/file_format/rails_development.rb
+- lib/request_log_analyzer/file_format/w3c.rb
 - lib/request_log_analyzer/filter.rb
 - lib/request_log_analyzer/filter/anonymize.rb
 - lib/request_log_analyzer/filter/field.rb
@@ -196,6 +199,7 @@ files:
 - spec/unit/file_format/delayed_job_format_spec.rb
 - spec/unit/file_format/file_format_api_spec.rb
 - spec/unit/file_format/format_autodetection_spec.rb
+- spec/unit/file_format/haproxy_format_spec.rb
 - spec/unit/file_format/line_definition_spec.rb
 - spec/unit/file_format/merb_format_spec.rb
 - spec/unit/file_format/mysql_format_spec.rb
@@ -204,6 +208,7 @@ files:
 - spec/unit/file_format/rack_format_spec.rb
 - spec/unit/file_format/rails3_format_spec.rb
 - spec/unit/file_format/rails_format_spec.rb
+- spec/unit/file_format/w3c_format_spec.rb
 - spec/unit/filter/anonymize_filter_spec.rb
 - spec/unit/filter/field_filter_spec.rb
 - spec/unit/filter/filter_spec.rb
@@ -277,6 +282,7 @@ test_files:
 - spec/unit/file_format/delayed_job_format_spec.rb
 - spec/unit/file_format/file_format_api_spec.rb
 - spec/unit/file_format/format_autodetection_spec.rb
+- spec/unit/file_format/haproxy_format_spec.rb
 - spec/unit/file_format/line_definition_spec.rb
 - spec/unit/file_format/merb_format_spec.rb
 - spec/unit/file_format/mysql_format_spec.rb
@@ -285,6 +291,7 @@ test_files:
 - spec/unit/file_format/rack_format_spec.rb
 - spec/unit/file_format/rails3_format_spec.rb
 - spec/unit/file_format/rails_format_spec.rb
+- spec/unit/file_format/w3c_format_spec.rb
 - spec/unit/filter/anonymize_filter_spec.rb
 - spec/unit/filter/field_filter_spec.rb
 - spec/unit/filter/filter_spec.rb