request-log-analyzer 1.9.10 → 1.11.0

data/.gitignore

@@ -8,4 +8,7 @@ requests.db
 /tmp
 /classes
 /files
-/coverage
+/coverage
+Gemfile.lock
+/.bundle/
+*.rbc

data/.infinity_test

@@ -0,0 +1,8 @@
+infinity_test do
+  
+  use :rubies => %w(1.8.7 1.9.2 ree), :test_framework => :rspec   # jruby rbx
+  
+  before(:each_ruby) do |environment|
+    environment.system('bundle install')
+  end
+end

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/bin/request-log-analyzer

@@ -58,7 +58,6 @@ begin
 
     command_line.switch(:debug)
     command_line.switch(:no_progress)
-    command_line.switch(:gets_memory_protection)
     command_line.switch(:silent)
 
     command_line.minimum_parameters = 1

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -1,4 +1,3 @@
-
 module RequestLogAnalyzer::Aggregator
 
   # The database aggregator will create an SQLite3 database with all parsed request information.
@@ -19,6 +18,8 @@ module RequestLogAnalyzer::Aggregator
     # Establishes a connection to the database and creates the necessary database schema for the
     # current file format
     def prepare
+      require 'request_log_analyzer/database'
+      
       @sources = {}
       @database = RequestLogAnalyzer::Database.new(options[:database])
       @database.file_format = source.file_format

data/lib/request_log_analyzer/aggregator.rb

@@ -1,9 +1,5 @@
 module RequestLogAnalyzer::Aggregator
 
-  autoload :Echo,             'request_log_analyzer/aggregator/echo'
-  autoload :Summarizer,       'request_log_analyzer/aggregator/summarizer'
-  autoload :DatabaseInserter, 'request_log_analyzer/aggregator/database_inserter'
-
   # The base class of an aggregator. This class provides the interface to which
   # every aggregator should comply (by simply subclassing this class).
   class Base
@@ -46,4 +42,8 @@ module RequestLogAnalyzer::Aggregator
     end
 
   end
-end
+end
+
+require 'request_log_analyzer/aggregator/echo'
+require 'request_log_analyzer/aggregator/summarizer'
+require 'request_log_analyzer/aggregator/database_inserter'

data/lib/request_log_analyzer/controller.rb

@@ -21,8 +21,6 @@ module RequestLogAnalyzer
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
     def self.build_from_arguments(arguments)
 
-      require 'mixins/gets_memory_protection' if arguments[:gets_memory_protection]
-
       options = {}
 
       # Copy fields
@@ -58,7 +56,6 @@ module RequestLogAnalyzer
 
       # Handle output format casing
       if options[:output].class == String
-        options[:output] = 'FancyHTML'  if options[:output] =~ /^fancy_?html$/i
         options[:output] = 'HTML'       if options[:output] =~ /^html$/i
         options[:output] = 'FixedWidth' if options[:output] =~ /^fixed_?width$/i
       end

data/lib/request_log_analyzer/database.rb

@@ -3,12 +3,7 @@ require 'active_record'
 
 class RequestLogAnalyzer::Database
 
-  autoload :Connection, 'request_log_analyzer/database/connection'
-  autoload :Base,       'request_log_analyzer/database/base'
-  autoload :Request,    'request_log_analyzer/database/request'
-  autoload :Source,     'request_log_analyzer/database/source'
-  autoload :Warning,    'request_log_analyzer/database/warning'
-
+  require 'request_log_analyzer/database/connection'
   include RequestLogAnalyzer::Database::Connection
 
   attr_accessor :file_format
@@ -100,5 +95,9 @@ class RequestLogAnalyzer::Database
       end
     end
   end
-
 end
+
+require 'request_log_analyzer/database/base'
+require 'request_log_analyzer/database/request'
+require 'request_log_analyzer/database/source'
+require 'request_log_analyzer/database/warning'

data/lib/request_log_analyzer/file_format/apache.rb

@@ -30,20 +30,23 @@ module RequestLogAnalyzer::FileFormat
 
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
-      '%' => { :regexp => '%', :captures => [] },
-      'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
-      'a' => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] },
-      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
-      'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
-      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
-      'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
-      'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
-      't' => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] },
-      's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
-      'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
-      'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
-      'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
-                       {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
+      '%' => { nil => { :regexp => '%', :captures => [] } },
+      'h' => { nil => { :regexp => "(#{hostname_or_ip_address})",  :captures => [{:name => :remote_host, :type => :string}] } },
+      'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
+      'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
+      'c' => { nil => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] } },
+      'D' => { nil     => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
+               'micro' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :musec }] },
+               'milli' => { :regexp => '(\d+|-)', :captures => [ {:name => :duration, :type => :duration, :unit => :msec }] }
+             },
+      'l' => { nil => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] } },
+      'T' => { nil => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] } },
+      't' => { nil => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] } },
+      's' => { nil => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] } },
+      'u' => { nil => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] } },
+      'U' => { nil => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] } },
+      'r' => { nil => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
+                       {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]} },
       'i' => { 'Referer'    => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
                'User-agent' => { :regexp => '(.*)',  :captures => [{:name => :user_agent, :type => :user_agent}] }
              }
@@ -72,14 +75,13 @@ module RequestLogAnalyzer::FileFormat
 
         if variable
           # Check if we recognize the log directive
-          directive = LOG_DIRECTIVES[variable]
-          directive = directive[arg] if directive && arg
+          directive = LOG_DIRECTIVES[variable][arg] rescue nil
 
           if directive
             line_regexp << directive[:regexp]   # Parse the value of the directive
             captures    += directive[:captures] # Add the directive's information to the captures
           else
-            puts "%#{directive} log directiven not yet supported, field is ignored."
+            puts "Apache log directive %#{arg}#{variable} is not yet supported by RLA, the field will be ignored."
             line_regexp << '.*' # Just accept any input for this literal
           end
         end
@@ -126,22 +128,11 @@ module RequestLogAnalyzer::FileFormat
         "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
 
-      # This function can be overridden to rewrite the path for better categorization in the
-      # reports.
-      def convert_path(value, definition)
-        value
-      end
-
       # This function can be overridden to simplify the user agent string for better
       # categorization in the reports
       def convert_user_agent(value, definition)
         value # TODO
       end
-
-      # Make sure that the string '-' is parsed as a nil value.
-      def convert_nillable_string(value, definition)
-        value == '-' ? nil : value
-      end
     end
   end
 end

data/lib/request_log_analyzer/file_format/delayed_job2.rb

@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_lock do |line|
       line.header = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] acquired lock on (\S+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
       
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_completed do |line|
       line.footer = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(\S+) pid:(\d+) completed after (\d+\.\d+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
       line.capture(:pid).as(:integer)

data/lib/request_log_analyzer/file_format/delayed_job21.rb

@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_lock do |line|
       line.header = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] acquired lock on (\S+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
       
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_completed do |line|
       line.footer = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
       line.capture(:pid).as(:integer)

data/lib/request_log_analyzer/file_format/haproxy.rb

@@ -0,0 +1,232 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Haproxy < RequestLogAnalyzer::FileFormat::Base
+
+    extend CommonRegularExpressions
+
+    # substitute version specific parts of the haproxy entry regexp.
+    def self.compose_regexp(millisecs, backends, counters, connections, queues)
+      %r{
+        (#{ip_address}):\d+\s # client_ip ':' client_port
+        \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})#{millisecs}\]\s # '[' accept_date ']'
+        (\S+)\s # frontend_name
+        #{backends}
+        #{counters}
+        (\d+)\s # status_code
+        \+?(\d+)\s # bytes_read
+        (\S+)\s # captured_request_cookie
+        (\S+)\s # captured_response_cookie
+        (\w|-)(\w|-)(\w|-)(\w|-)\s # termination_state
+        #{connections}
+        #{queues}
+        (\S*)\s? # captured_request_headers
+        (\S*)\s? # captured_response_headers
+        "([^"]*)" # '"' http_request '"'
+      }x
+    end
+
+    # Define line types
+
+    # line definition for haproxy 1.3 and higher
+    line_definition :haproxy13 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /\.\d{3}\] \S+ \S+\/\S+ / # .millisecs] frontend_name backend_name/server_name
+
+      line.regexp = compose_regexp(
+        '\.\d{3}', # millisecs
+        '(\S+)\/(\S+)\s', # backend_name '/' server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s', # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
+        '(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
+      )
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:backend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tw).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:actconn).as(:integer)
+      line.capture(:feconn).as(:integer)
+      line.capture(:beconn).as(:integer)
+      line.capture(:srv_conn).as(:integer)
+      line.capture(:retries).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
+    # haproxy 1.2 has a few fields less than 1.3+
+    line_definition :haproxy12 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /\.\d{3}\] \S+ \S+ / # .millisecs] frontend_name server_name
+
+      line.regexp = compose_regexp(
+        '\.\d{3}', # millisecs
+        '(\S+)\s', # server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\/(\d+)\s', # srv_conn '/' listener_conn '/' process_conn
+        '(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
+      )
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tw).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:srv_conn).as(:integer)
+      line.capture(:listener_conn).as(:integer)
+      line.capture(:process_conn).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
+    # and haproxy 1.1 has even less fields
+    line_definition :haproxy11 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /:\d{2}\] \S+ \S+ / # :secs] frontend_name server_name
+
+      line.regexp = compose_regexp(
+        '', # no millisec precision in this version of haproxy
+        '(\S+)\s', # server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\s', # listener_conn '/' process_conn
+        '' # no queues in this version of haproxy
+      )
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:listener_conn).as(:integer)
+      line.capture(:process_conn).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
+    # Define the summary report
+    report do |analyze|
+      analyze.hourly_spread :field => :timestamp
+
+      analyze.frequency :client_ip,
+        :title => "Hits per IP"
+
+      analyze.frequency :frontend_name,
+        :title => "Hits per frontend service"
+
+      analyze.frequency :backend_name,
+        :title => "Hits per backend service"
+
+      analyze.frequency :server_name,
+        :title => "Hits per backend server"
+
+      analyze.frequency :status_code,
+        :title => "HTTP response code frequency"
+
+      analyze.frequency :http_request,
+        :title => "Most popular requests"
+
+      analyze.frequency :http_request,
+        :title => "Most frequent HTTP 40x errors",
+        :category => lambda { |r| "#{r[:http_request]}"},
+        :if => lambda { |r| r[:status_code] >= 400 and r[:status_code] <= 417 }
+
+      analyze.frequency :http_request,
+        :title => "Most frequent HTTP 50x errors",
+        :category => lambda { |r| "#{r[:http_request]}"},
+        :if => lambda { |r| r[:status_code] >= 500 and r[:status_code] <= 505 }
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per frontend service",
+        :category => lambda { |r| "#{r[:frontend_name]}"}
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per backend service",
+        :category => lambda { |r| "#{r[:backend_name]}"}
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per backend server",
+        :category => lambda { |r| "#{r[:server_name]}"}
+
+      analyze.duration :tr,
+        :title => "Time waiting for backend response",
+        :category => lambda { |r| "#{r[:http_request]}"}
+
+      analyze.duration :tt,
+        :title => "Total time spent on request",
+        :category => lambda { |r| "#{r[:http_request]}"}
+    end
+
+    # Define a custom Request class for the HAProxy file format to speed up
+    # timestamp handling. Shamelessly copied from apache.rb
+    class Request < RequestLogAnalyzer::Request
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return
+      # a integer to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
+      end
+
+      # Make sure that the strings '-' or '{}' or '' are parsed as a nil value.
+      def convert_nillable_string(value, definition)
+        value =~ /-|\{\}|^$/ ? nil : value
+      end
+
+      # Make sure that -1 is parsed as a nil value.
+      def convert_nillable_duration(value, definition)
+        value == '-1' ? nil : convert_duration(value, definition)
+      end
+
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/mysql.rb

@@ -15,7 +15,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :user_host do |line|
       line.header = :alternative
       line.teaser = /\# User\@Host\: /
-      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[(#{ip_address(true)})\]/
+      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ (#{hostname(true)}) \[(#{ip_address(true)})\]/
       
       line.capture(:user)
       line.capture(:host)
@@ -34,18 +34,18 @@ module RequestLogAnalyzer::FileFormat
     end
 
     line_definition :use_database do |line|
-      line.regexp   = /^use (\w+);\s*$/
+      line.regexp   = /^\s*use (\w+);\s*$/
       line.capture(:database)
     end
 
     line_definition :query_part do |line|
-      line.regexp   = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
+      line.regexp   = /^\s*(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
       line.capture(:query_fragment)
     end
 
     line_definition :query do |line|
       line.footer = true
-      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
+      line.regexp = /^(?!\s*(?:use |\# |SET ))(.*);\s*$/
       line.capture(:query).as(:sql)
     end
 
@@ -84,7 +84,7 @@ module RequestLogAnalyzer::FileFormat
         sql.gsub!(/(:int,)+:int/, ':ints')                                # replace multiple ints by a list
         sql.gsub!(/(:string,)+:string/, ':strings')                       # replace multiple strings by a list
 
-        return sql.rstrip
+        return sql.strip
       end
 
       def host

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -23,12 +23,19 @@ module RequestLogAnalyzer::FileFormat
     # Processing by QueriesController#index as HTML
     line_definition :processing do |line|
       line.teaser = /Processing by /
-      line.regexp = /Processing by ([A-Za-z0-9\-:]+)\#(\w+) as ([\w\/\*]+)/
+      line.regexp = /Processing by ([A-Za-z0-9\-:]+)\#(\w+) as ([\w\/\*]*)/
       
       line.capture(:controller)
       line.capture(:action)
       line.capture(:format)
     end
+
+    # Parameters: {"action"=>"cached", "controller"=>"cached"}
+    line_definition :parameters do |line|
+      line.teaser   = /  Parameters:/
+      line.regexp   = /  Parameters:\s+(\{.*\})/
+      line.capture(:params).as(:eval)
+    end
     
     # Completed 200 OK in 224ms (Views: 200.2ms | ActiveRecord: 3.4ms)
     # Completed 302 Found in 23ms
@@ -36,7 +43,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :completed do |line|
       line.footer = true
       line.teaser = /Completed /
-      line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms\))?/
+      line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms.*\))?/
       
       line.capture(:status).as(:integer)
       line.capture(:duration).as(:duration, :unit => :msec)

data/lib/request_log_analyzer/file_format/w3c.rb

@@ -0,0 +1,56 @@
+module RequestLogAnalyzer::FileFormat
+
+  # FileFormat for W3C access logs.
+  class W3c < Base
+    
+    extend CommonRegularExpressions
+
+    line_definition :access do |line|
+      line.header = true
+      line.footer = true
+      line.regexp = /^(#{timestamp('%Y-%m-%d %H:%M:%S')}) (#{ip_address}) (.*) (#{ip_address}) (\d+) (\w+) ([\w|\/|.]+) \- (\d+) (\d+) (\d+) (\d+) (.*) (\S+)/
+      
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:remote_ip)
+      line.capture(:username).as(:nillable_string)
+      line.capture(:local_ip)
+      line.capture(:port).as(:integer)
+      line.capture(:method)
+      line.capture(:path).as(:path)
+      line.capture(:http_status).as(:integer)
+      line.capture(:bytes_sent).as(:traffic, :unit => :byte)
+      line.capture(:bytes_received).as(:traffic, :unit => :byte)
+      line.capture(:duration).as(:duration, :unit => :msec)
+      line.capture(:user_agent)
+      line.capture(:referer)
+    end
+
+    report do |analyze|
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :category => :http_method, :title => "HTTP methods"
+      analyze.frequency :category => :http_status, :title => "HTTP statuses"
+      
+      analyze.frequency :category => :path, :title => "Most popular URIs"
+
+      analyze.frequency :category => :user_agent, :title => "User agents"
+      analyze.frequency :category => :referer,    :title => "Referers"
+
+      analyze.duration :duration => :duration,  :category => :path, :title => 'Request duration'
+      analyze.traffic  :traffic => :bytes_sent, :category => :path, :title => 'Traffic out'
+      analyze.traffic  :traffic => :bytes_received, :category => :path, :title => 'Traffic in'
+    end
+    
+    class Request < RequestLogAnalyzer::Request
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
+      # to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[0,4]}#{value[5,2]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
+      end
+    end
+    
+
+  end
+  
+end