request-log-analyzer 1.0.2 → 1.6.3

data/lib/request_log_analyzer/database.rb

@@ -0,0 +1,102 @@
+require 'rubygems'
+require 'active_record'
+
+class RequestLogAnalyzer::Database
+
+  def self.const_missing(const) # :nodoc:
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end
+
+  include RequestLogAnalyzer::Database::Connection
+
+  attr_accessor :file_format
+  attr_reader :line_classes
+
+  def initialize(connection_identifier = nil)
+    @line_classes = []
+    RequestLogAnalyzer::Database::Base.database = self
+    connect(connection_identifier)
+  end
+
+  # Returns the ORM class for the provided line type
+  def get_class(line_type)
+    line_type = line_type.name if line_type.respond_to?(:name)
+    Object.const_get("#{line_type}_line".camelize)
+  end
+
+  def default_classes
+    [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
+  end
+
+  # Loads the ORM classes by inspecting the tables in the current database
+  def load_database_schema!
+    connection.tables.map do |table|
+      case table.to_sym
+      when :warnings then RequestLogAnalyzer::Database::Warning
+      when :sources  then RequestLogAnalyzer::Database::Source
+      when :requests then RequestLogAnalyzer::Database::Request
+      else load_activerecord_class(table)
+      end
+    end
+  end
+
+  # Returns an array of all the ActiveRecord-bases ORM classes for this database
+  def orm_classes
+    default_classes + line_classes
+  end
+
+  # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
+  # a table name or a LineDefinition instance.
+  def load_activerecord_class(linedefinition_or_table)
+
+    case linedefinition_or_table
+    when String, Symbol
+      klass_name = linedefinition_or_table.to_s.singularize.camelize
+      klass      = RequestLogAnalyzer::Database::Base.subclass_from_table(linedefinition_or_table)
+    when RequestLogAnalyzer::LineDefinition
+      klass_name = "#{linedefinition_or_table.name}_line".camelize
+      klass      = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
+    end
+
+    Object.const_set(klass_name, klass)
+    klass = Object.const_get(klass_name)
+    @line_classes << klass
+    return klass
+  end
+
+  def fileformat_classes
+    raise "No file_format provided!" unless file_format
+    line_classes    = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
+    return default_classes + line_classes
+  end
+
+  # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
+  # file format definition. These ORM classes will later be used to create records in the database.
+  def create_database_schema!
+    fileformat_classes.each { |klass| klass.create_table! }
+  end
+
+  # Drops the table of all the ORM classes, and unregisters the classes
+  def drop_database_schema!
+    file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
+    remove_orm_classes!
+  end
+
+  # Registers the default ORM classes in the default namespace
+  def register_default_orm_classes!
+    Object.const_set('Request', RequestLogAnalyzer::Database::Request)
+    Object.const_set('Source',  RequestLogAnalyzer::Database::Source)
+    Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
+  end
+
+  # Unregisters every ORM class constant
+  def remove_orm_classes!
+    orm_classes.each do |klass|
+      if klass.respond_to?(:name) && !klass.name.blank?
+        klass_name = klass.name.split('::').last
+        Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name)
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -0,0 +1,74 @@
+module RequestLogAnalyzer::FileFormat
+
+  # FileFormat for Amazon S3 access logs.
+  #
+  # Access logs are disabled by default on Amazon S3. To enable logging, see
+  # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
+  class AmazonS3 < Base
+    
+    extend CommonRegularExpressions
+
+    line_definition :access do |line|
+      line.header = true
+      line.footer = true
+      line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]*)" "([^"]*)"/
+      
+      line.capture(:bucket_owner)
+      line.capture(:bucket)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:remote_ip)
+      line.capture(:requester)
+      line.capture(:request_id)
+      line.capture(:operation)
+      line.capture(:key).as(:nillable_string)
+      line.capture(:request_uri)
+      line.capture(:http_status).as(:integer)
+      line.capture(:error_code).as(:nillable_string)
+      line.capture(:bytes_sent).as(:traffic, :unit => :byte)
+      line.capture(:object_size).as(:traffic, :unit => :byte)
+      line.capture(:total_time).as(:duration, :unit => :msec)
+      line.capture(:turnaround_time).as(:duration, :unit => :msec)
+      line.capture(:referer).as(:referer)
+      line.capture(:user_agent).as(:user_agent)
+    end
+
+    report do |analyze|
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
+      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
+      analyze.traffic  :traffic => :bytes_sent,  :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
+      analyze.frequency :category => :http_status, :title => 'HTTP status codes'
+      analyze.frequency :category => :error_code, :title => 'Error codes'
+    end
+
+    class Request < RequestLogAnalyzer::Request
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
+      # to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
+      end
+
+      # Make sure that the string '-' is parsed as a nil value.
+      def convert_nillable_string(value, definition)
+        value == '-' ? nil : value
+      end
+
+      # Can be implemented in subclasses for improved categorizations
+      def convert_referer(value, definition)
+        value == '-' ? nil : value
+      end
+
+      # Can be implemented in subclasses for improved categorizations
+      def convert_user_agent(value, definition)
+        value == '-' ? nil : value
+      end
+    end
+
+  end
+end

data/lib/request_log_analyzer/file_format/apache.rb

@@ -0,0 +1,147 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The Apache file format is able to log Apache access.log files.
+  #
+  # The access.log can be configured in Apache to have many different formats. In theory, this
+  # FileFormat can handle any format, but it must be aware of the log formatting that is used
+  # by sending the formatting string as parameter to the create method, e.g.:
+  #
+  #     RequestLogAnalyzer::FileFormat::Apache.create('%h %l %u %t "%r" %>s %b')
+  #
+  # It also supports the predefined Apache log formats "common" and "combined". The line
+  # definition and the report definition will be constructed using this file format string.
+  # From the command line, you can provide the format string using the <tt>--apache-format</tt>
+  # command line option.
+  class Apache < Base
+
+    extend CommonRegularExpressions
+
+    # A hash of predefined Apache log formats
+    LOG_FORMAT_DEFAULTS = {
+      :common   => '%h %l %u %t "%r" %>s %b',
+      :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
+      :rack     => '%h %l %u %t "%r" %>s %b %T',
+      :referer  => '%{Referer}i -> %U',
+      :agent    => '%{User-agent}i'
+    }
+
+    # I have encountered two timestamp types, with timezone and without. Parse both.
+    APACHE_TIMESTAMP = Regexp.union(timestamp('%d/%b/%Y:%H:%M:%S %z'), timestamp('%d/%b/%Y %H:%M:%S'))
+
+    # A hash that defines how the log format directives should be parsed.
+    LOG_DIRECTIVES = {
+      '%' => { :regexp => '%', :captures => [] },
+      'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
+      'a' => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] },
+      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
+      'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
+      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
+      'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
+      'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
+      't' => { :regexp => "\\[(#{APACHE_TIMESTAMP})?\\]", :captures => [{:name => :timestamp, :type => :timestamp}] },
+      's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
+      'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
+      'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
+      'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
+                       {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
+      'i' => { 'Referer'    => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
+               'User-agent' => { :regexp => '(.*)',  :captures => [{:name => :user_agent, :type => :user_agent}] }
+             }
+    }
+
+    # Creates the Apache log format language based on a Apache log format string.
+    # It will set up the line definition and the report trackers according to the Apache access log format,
+    # which should be passed as first argument. By default, is uses the 'combined' log format.
+    def self.create(*args)
+      access_line = access_line_definition(args.first)
+      trackers = report_trackers(access_line) + report_definer.trackers
+      self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
+    end
+
+    # Creates the access log line definition based on the Apache log format string
+    def self.access_line_definition(format_string)
+      format_string ||= :common
+      format_string   = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
+      
+      
+      line_regexp = ''
+      captures    = []
+      format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
+        
+        line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
+
+        if variable
+          # Check if we recognize the log directive
+          directive = LOG_DIRECTIVES[variable]
+          directive = directive[arg] if directive && arg
+
+          if directive
+            line_regexp << directive[:regexp]   # Parse the value of the directive
+            captures    += directive[:captures] # Add the directive's information to the captures
+          else
+            puts "%#{directive} log directiven not yet supported, field is ignored."
+            line_regexp << '.*' # Just accept any input for this literal
+          end
+        end
+      end
+      
+      # Return a new line definition object
+      return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
+                                        :captures => captures, :header => true, :footer => true)
+    end
+
+    # Sets up the report trackers according to the fields captured by the access line definition.
+    def self.report_trackers(line_definition)
+      analyze = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
+
+      analyze.timespan      if line_definition.captures?(:timestamp)
+      analyze.hourly_spread if line_definition.captures?(:timestamp)
+
+      analyze.frequency :category => :http_method, :title => "HTTP methods"  if line_definition.captures?(:http_method)
+      analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
+      analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs"    if line_definition.captures?(:path)
+
+      analyze.frequency :category => :user_agent, :title => "User agents"    if line_definition.captures?(:user_agent)
+      analyze.frequency :category => :referer,    :title => "Referers"       if line_definition.captures?(:referer)
+
+      analyze.duration :duration => :duration,  :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
+      analyze.traffic  :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic'          if line_definition.captures?(:bytes_sent)
+
+      return analyze.trackers
+    end
+
+    # Define a custom Request class for the Apache file format to speed up timestamp handling.
+    class Request < RequestLogAnalyzer::Request
+
+      def category
+        first(:path)
+      end
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
+      # to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
+      end
+
+      # This function can be overridden to rewrite the path for better categorization in the
+      # reports.
+      def convert_path(value, definition)
+        value
+      end
+
+      # This function can be overridden to simplify the user agent string for better
+      # categorization in the reports
+      def convert_user_agent(value, definition)
+        value # TODO
+      end
+
+      # Make sure that the string '-' is parsed as a nil value.
+      def convert_nillable_string(value, definition)
+        value == '-' ? nil : value
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/delayed_job.rb

@@ -0,0 +1,55 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The DelayedJob file format parsed log files that are created by DelayedJob.
+  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
+  class DelayedJob < Base
+    
+    line_definition :job_lock do |line|
+      line.header = true
+      line.regexp = /\* \[JOB\] acquiring lock on (\S+)/
+      
+      line.capture(:job)
+    end
+    
+    line_definition :job_completed do |line|
+      line.footer = true
+      line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/
+
+      line.capture(:completed_job)
+      line.capture(:duration).as(:duration, :unit => :sec) 
+    end
+    
+    line_definition :job_failed do |line|
+      line.footer = true
+      line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/
+      
+      line.capture(:failed_job)
+      line.capture(:exception)
+      line.capture(:attempts).as(:integer)
+    end
+    
+    line_definition :job_lock_failed do |line|
+      line.footer = true
+      line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/
+      
+      line.capture(:locked_job)
+    end
+    
+    # line_definition :batch_completed do |line|
+    #   line.header = true
+    #   line.footer = true
+    #   line.regexp = /(\d+) jobs processed at (\d+\.\d+) j\/s, (\d+) failed .../
+    #
+    #   line.capture(:total_amount).as(:integer)
+    #   line.capture(:mean_duration).as(:duration, :unit => :sec)
+    #   line.capture(:failed_amount).as(:integer)
+    # end
+    
+    report do |analyze|
+      analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
+      analyze.frequency :job, :if => lambda { |request| request[:attempts] ==  1 }, :title => "Failed jobs"
+      
+      analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/merb.rb

@@ -1,33 +1,69 @@
-module RequestLogAnalyzer::FileFormat::Merb
+module RequestLogAnalyzer::FileFormat
+
+  # The Merb file format parses the request header with the timestamp, the params line
+  # with the most important request information and the durations line which contains
+  # the different request durations that can be used for analysis.
+  class Merb < Base
+
+    extend CommonRegularExpressions
 
-  LINE_DEFINITIONS = {
-    
     # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
-    :started => {
-      :header   => true,
-      :teaser   => /Started/,
-      :regexp   => /Started request handling\:\ (.+)/,
-      :captures => [{ :name => :timestamp, :type => :timestamp, :anonymize => :slightly }]
-    },
-    
+    line_definition :started do |line|
+      line.header = true
+      line.teaser = /Started request handling\:/
+      line.regexp = /Started request handling\:\ (#{timestamp('%a %b %d %H:%M:%S %z %Y')})/
+      line.captures << { :name => :timestamp, :type => :timestamp }
+    end
+
     # ~ Params: {"action"=>"create", "controller"=>"session"}
-    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
-    :params => {
-      :teaser   => /Params/,
-      :regexp   => /Params\:\ \{(.+)\}/,
-      :captures => [{ :name => :raw_hash, :type => :string}]
-    },
-    
+    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy"}
+    line_definition :params do |line|
+      line.teaser = /Params\:\ /
+      line.regexp = /Params\:\ (\{.+\})/
+      line.captures << { :name => :params, :type => :eval, :provides => {
+            :namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
+    end
+
     # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
-    :completed => {
-      :footer   => true,
-      :teaser   => /\{:dispatch_time/,
-      :regexp   => /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/,
-      :captures => [{ :name => :dispatch_time,       :type => :sec, :anonymize => :slightly }, 
-                    { :name => :after_filters_time,  :type => :sec, :anonymize => :slightly }, 
-                    { :name => :before_filters_time, :type => :sec, :anonymize => :slightly }, 
-                    { :name => :action_time,         :type => :sec, :anonymize => :slightly }]
-    }
-  }
-    
-end
+    line_definition :completed do |line|
+      line.footer = true
+      # line.teaser = Regexp.new(Regexp.quote('~ {:'))
+      line.regexp = /(\{.*\:dispatch_time\s*=>\s*\d+\.\d+.*\})/
+      line.captures << { :name => :times_hash, :type => :eval, :provides => {
+            :dispatch_time => :duration, :after_filters_time => :duration,
+            :before_filters_time => :duration, :action_time => :duration } }
+    end
+
+    REQUEST_CATEGORIZER = Proc.new do |request|
+      category = "#{request[:controller]}##{request[:action]}"
+      category = "#{request[:namespace]}::#{category}" if request[:namespace]
+      category = "#{category}.#{request[:format]}"     if request[:format]
+      category
+    end
+
+    report do |analyze|
+
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => "Top 20 by hits"
+      analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
+
+      # analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
+      # analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
+      # analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
+    end
+
+    class Request < RequestLogAnalyzer::Request
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Speed up timestamp conversion
+      def convert_timestamp(value, definition)
+        "#{value[26,4]}#{MONTHS[value[4,3]]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/file_format/mysql.rb

@@ -0,0 +1,101 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Mysql < Base
+
+    extend CommonRegularExpressions
+
+    line_definition :time do |line|
+      line.header = :alternative
+      line.teaser = /\# Time: /
+      line.regexp = /\# Time: (#{timestamp('%y%m%d %k:%M:%S')})/
+      
+      line.capture(:timestamp).as(:timestamp)
+    end
+
+    line_definition :user_host do |line|
+      line.header = :alternative
+      line.teaser = /\# User\@Host\: /
+      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[(#{ip_address(true)})\]/
+      
+      line.capture(:user)
+      line.capture(:host)
+      line.capture(:ip)
+    end
+
+    line_definition :query_statistics do |line|
+      line.header = :alternative
+      line.teaser = /\# Query_time: /
+      line.regexp = /\# Query_time: (\d+(?:\.\d+)?)\s+Lock_time: (\d+(?:\.\d+)?)\s+Rows_sent: (\d+)\s+Rows_examined: (\d+)/
+      
+      line.capture(:query_time).as(:duration, :unit => :sec)
+      line.capture(:lock_time).as(:duration, :unit => :sec)
+      line.capture(:rows_sent).as(:integer)
+      line.capture(:rows_examined).as(:integer)
+    end
+
+    line_definition :use_database do |line|
+      line.regexp   = /^use (\w+);\s*$/
+      line.capture(:database)
+    end
+
+    line_definition :query_part do |line|
+      line.regexp   = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
+      line.capture(:query_fragment)
+    end
+
+    line_definition :query do |line|
+      line.footer = true
+      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
+      line.capture(:query).as(:sql)
+    end
+
+    PER_USER       = :user
+    PER_QUERY      = :query
+    PER_USER_QUERY = Proc.new { |request| "#{request[:user]}@#{request.host}: #{request[:query]}" }
+
+    report do |analyze|
+      analyze.timespan :line_type => :time
+      analyze.frequency :user, :title => "Users with most queries"
+      analyze.duration :query_time, :category => PER_USER, :title => 'Query time per user'
+      analyze.duration :query_time, :category => PER_USER_QUERY, :title => 'Query time'
+      
+      analyze.duration :lock_time,  :category => PER_USER_QUERY, :title => 'Lock time',
+                       :if => lambda { |request| request[:lock_time] > 0.0 }
+      
+      analyze.numeric_value :rows_examined, :category => PER_USER_QUERY, :title => "Rows examined"
+      analyze.numeric_value :rows_sent,     :category => PER_USER_QUERY, :title => "Rows sent"
+    end
+  
+    class Request < RequestLogAnalyzer::Request
+
+      def convert_sql(value, definition)
+
+        # Recreate the full SQL query by joining all the previous parts and this last line
+        sql = every(:query_fragment).join("\n") + value
+
+        # Sanitize an SQL query so that it can be used as a category field.
+        # sql.gsub!(/\/\*.*\*\//, '')                                       # remove comments
+        sql.gsub!(/\s+/, ' ')                                             # remove excessive whitespace
+        sql.gsub!(/`([^`]+)`/, '\1')                                      # remove quotes from field names
+        sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date')                         # replace dates
+        sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime')   # replace timestamps
+        sql.gsub!(/'[^']*'/, ':string')                                   # replace strings
+        sql.gsub!(/\b\d+\b/, ':int')                                      # replace integers
+        sql.gsub!(/(:int,)+:int/, ':ints')                                # replace multiple ints by a list
+        sql.gsub!(/(:string,)+:string/, ':strings')                       # replace multiple strings by a list
+
+        return sql.rstrip
+      end
+
+      def host
+        self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host]
+      end
+
+      # Convert the timestamp to an integer
+      def convert_timestamp(value, definition)
+        all,y,m,d,h,i,s = value.split(/(\d\d)(\d\d)(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
+        ('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/postgresql.rb

@@ -0,0 +1,68 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Postgresql < Base
+
+    extend CommonRegularExpressions
+    
+    line_definition :query do |line|
+      line.header = true
+      line.teaser = /LOG\:  query\:/
+      line.regexp = /(#{timestamp('%y-%m-%d %k:%M:%S')})\ LOG:  query:\s+(.*)/
+
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:query_fragment)
+    end
+      
+    line_definition :duration do |line|
+      line.footer = true
+      line.teaser = /duration:/
+      line.regexp = /#{timestamp('%y-%m-%d %k:%M:%S')}\ LOG\:  duration\: (.*)(\ )sec/
+
+      line.capture(:query_time).as(:duration, :unit => :sec)
+      line.capture(:query).as(:sql) # Hack to gather up fragments
+    end
+    
+    line_definition :query_fragment do |line|
+      line.regexp = /^(?!.*LOG)\s*(.*)\s*/
+      line.capture(:query_fragment)
+    end
+
+    report do |analyze|
+      analyze.timespan
+      analyze.hourly_spread      
+      analyze.duration :query_time, :category => :query, :title => 'Query time'
+    end
+  
+    class Request < RequestLogAnalyzer::Request
+
+      def convert_sql(value, definition)
+        
+        # Recreate the full SQL query by joining all the previous parts and this last line
+        sql = every(:query_fragment).join("\n") + value
+
+        # Sanitize an SQL query so that it can be used as a category field.
+        # sql.gsub!(/\/\*.*\*\//, '')                                       # remove comments
+        sql.gsub!(/\s+/, ' ')                                             # remove excessive whitespace
+        sql.gsub!(/"([^"]+)"/, '\1')                                      # remove quotes from field names
+        sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date')                         # replace dates
+        sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime')   # replace timestamps
+        sql.gsub!(/'[^']*'/, ':string')                                   # replace strings
+        sql.gsub!(/\b\d+\b/, ':int')                                      # replace integers
+        sql.gsub!(/(:int,)+:int/, ':ints')                                # replace multiple ints by a list
+        sql.gsub!(/(:string,)+:string/, ':strings')                       # replace multiple strings by a list
+
+        return sql.lstrip.rstrip
+      end
+
+      def host
+        self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host]
+      end
+
+      # Convert the timestamp to an integer
+      def convert_timestamp(value, definition)
+        all,y,m,d,h,i,s = value.split(/(\d\d)-(\d\d)-(\d\d)\s+(\d?\d):(\d\d):(\d\d)/)
+        ('20%s%s%s%s%s%s' % [y,m,d,h.rjust(2, '0'),i,s]).to_i
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/rack.rb

@@ -0,0 +1,9 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Rack < Apache
+
+    def self.create(*args)
+      super(:rack, *args)
+    end
+  end
+end