request-log-analyzer 1.1.2 → 1.2.0

data/DESIGN

@@ -4,21 +4,35 @@ This allows you to easily add extra reports, filters and outputs.
 
 1) Build pipeline.
                              -> Aggregator  (database)
-  Source -> Filter -> Filter -> Aggregator  (summary report)
+  Source -> Filter -> Filter -> Aggregator  (summary report)  -> Output
                              -> Aggregator  (...)
-
+                             
 2) Start chunk producer and push chunks through pipeline.
-  Controller.start
+ Controller.start
+ 
+RequestLogAnalyzer::Source is an Object that pushes requests into the chain.
+At the moment you can only use the log-parser as a source.
+It accepts files or stdin and can parse then into request objects using a RequestLogAnalyzer::FileFormat definition.
+In the future we want to be able to have a generated request database as source as this will make interactive
+down drilling possible.
+
+The filters are all subclasses of the RequestLogAnalyzer::Filter class.
+They accept a request object, manipulate or drop it, and then pass the request object on to the next filter
+in the chain.
+At the moment there are three types of filters available: Anonymize, Field and Timespan.
+
+The Aggregators all inherit from the RequestLogAnalyzer::Aggregator class.
+All the requests that come out of the Filterchain are fed into all the aggregators in parallel.
+These aggregators can do anything what they want with the given request.
+For example: the Database aggregator will just store all the requests into a SQLite database while the Summarizer will
+generate a wide range of statistical reports from them.
 
 3) Gather output from pipeline.
   Controller.report
 
-At the moment the supported sources are file and STDIN.
-In the future we want to be able to have a generated request database as source.
-This will make interactive downdrilling possible.
-
+All Aggregators are asked to report what they have done. For example the database will report: I stuffed x requests
+into SQLite database Y. The Summarizer will output its reports.
 
-For the report generation output we now use the File and the STDOUT class, as they both support <<.
-In the future we want to have a OutputFile, OutputSTDOUT and OutputHTML class, so that reports can generate
-tables, lines and comments and push them into the output class.
+The output is pushed to a RequestLogAnalyzer::Output object, which takes care of the output.
+It can generate either ASCII, UTF8 or even HTML output.
 

data/README.rdoc

@@ -3,7 +3,7 @@
 This is a simple command line tool to analyze request log files of both Rails and
 Merb to produce a performance report. Its purpose is to find what actions are best candidates for optimization.
 
-* Analyzes Rails log files (all versions)
+* Analyzes Rails log files (all versions), Merb logs, or any other log format
 * Can combine multiple files (handy if you are using logrotate)
 * Uses several metrics, including cumulative request time, average request time, process blockers, database and rendering time, HTTP methods and statuses, Rails action cache statistics, etc.) (Sample output: http://wiki.github.com/wvanbergen/request-log-analyzer/sample-output)
 * Low memory footprint (server-safe)

data/Rakefile

@@ -1,4 +1,3 @@
 Dir[File.dirname(__FILE__) + "/tasks/*.rake"].each { |file| load(file) }
  
-task :default => :spec
-
+task :default => :spec

data/bin/request-log-analyzer

@@ -1,15 +1,8 @@
 #!/usr/bin/ruby
 require File.dirname(__FILE__) + '/../lib/request_log_analyzer'
 require File.dirname(__FILE__) + '/../lib/cli/command_line_arguments'
-
-def terminal_width(default = 81)
-  IO.popen('stty -a') do |pipe|
-    column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
-    width = column_line ? $1.to_i : default
-  end
-rescue 
-  default  
-end
+require File.dirname(__FILE__) + '/../lib/cli/progressbar'
+require File.dirname(__FILE__) + '/../lib/cli/tools'
 
 # Parse the arguments given via commandline
 begin
@@ -29,7 +22,7 @@ begin
     
     command_line.option(:format, :alias => :f, :default => 'rails')
     command_line.option(:file, :alias => :e)
-    command_line.switch(:assume_correct_order)
+    command_line.option(:parse_strategy, :default => 'assume-correct')
     
     command_line.option(:aggregator, :alias => :a, :multiple => true)
     command_line.option(:database, :alias => :d)    
@@ -50,6 +43,7 @@ begin
   end
   
 rescue CommandLine::Error => e
+  puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"  
   puts "ARGUMENT ERROR: " + e.message if e.message
   puts
   puts "Usage: request-log-analyzer [LOGFILES*] <OPTIONS>"
@@ -80,23 +74,6 @@ rescue CommandLine::Error => e
   exit(0) 
 end
 
-def install_rake_tasks(install_type)
-  if install_type == 'rails'
-    require 'ftools'
-    if File.directory?('./lib/tasks/')
-      File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
-      puts "Installed rake tasks."
-      puts "To use, run: rake log:analyze"
-    else
-      puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
-      puts "Installation aborted."
-    end
-  else
-    raise "Cannot perform this install type! (#{install_type})"
-  end
-end
-
-
 case arguments.command
 when :install
   install_rake_tasks(arguments.parameters[0])

data/lib/cli/progressbar.rb

@@ -119,23 +119,6 @@ module CommandLine
       end
     end
 
-    def get_width
-      # FIXME: I don't know how portable it is.
-      default_width = 80
-      begin
-        tiocgwinsz = 0x5413
-        data = [0, 0, 0, 0].pack("SSSS")
-        if @out.ioctl(tiocgwinsz, data) >= 0 then
-          rows, cols, xpixels, ypixels = data.unpack("SSSS")
-          if cols >= 0 then cols else default_width end
-        else
-          default_width
-        end
-      rescue Exception
-        default_width
-      end
-    end
-
     def show
       arguments = @format_arguments.map {|method| 
         method = sprintf("fmt_%s", method)
@@ -143,7 +126,7 @@ module CommandLine
       }
       line = sprintf(@format, *arguments)
 
-      width = get_width
+      width = terminal_width(80)
       if line.length == width - 1 
         @out.print(line + eol)
         @out.flush
@@ -176,7 +159,7 @@ module CommandLine
     public
     def clear
       @out.print "\r"
-      @out.print(" " * (get_width - 1))
+      @out.print(" " * (terminal_width(80) - 1))
       @out.print "\r"
     end
 

data/lib/cli/tools.rb

@@ -0,0 +1,46 @@
+# Try to determine the terminal with.
+# If it is not possible to to so, it returns the default_width.
+# <tt>default_width</tt> Defaults to 81
+def terminal_width(default_width = 81)
+  tiocgwinsz = 0x5413
+  data = [0, 0, 0, 0].pack("SSSS")
+  if @out.ioctl(tiocgwinsz, data) >= 0 
+    rows, cols, xpixels, ypixels = data.unpack("SSSS")
+    raise unless cols > 0
+    cols
+  else
+    raise
+  end
+rescue   
+  begin 
+    IO.popen('stty -a') do |pipe|
+      column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
+      raise unless column_line
+      $1.to_i
+    end
+  rescue
+    default_width
+  end
+end
+
+# Copies request-log-analyzer analyzer rake tasks into the /lib/tasks folder of a project, for easy access and
+# environment integration.
+# <tt>install_type</tt> Type of project to install into. Defaults to :rails.
+# Raises if it cannot find the project folder or if the install_type is now known.
+def install_rake_tasks(install_type = :rails)
+  if install_type.to_sym == :rails
+    require 'ftools'
+    if File.directory?('./lib/tasks/')
+      File.copy(File.dirname(__FILE__) + '/../tasks/request_log_analyzer.rake', './lib/tasks/request_log_analyze.rake')
+      puts "Installed rake tasks."
+      puts "To use, run: rake log:analyze"
+    else
+      puts "Cannot find /lib/tasks folder. Are you in your Rails directory?"
+      puts "Installation aborted."
+    end
+  else
+    raise "Cannot perform this install type! (#{install_type.to_s})"
+  end
+end
+
+

data/lib/request_log_analyzer/aggregator/database.rb

@@ -30,7 +30,8 @@ module RequestLogAnalyzer::Aggregator
     def aggregate(request)
       @request_object = @request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
       request.lines.each do |line|
-        attributes = line.reject { |k, v| [:line_type].include?(k) }
+        class_columns = @orm_module.const_get("#{line[:line_type]}_line".classify).column_names.reject { |column| ['id'].include?(column) }
+        attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
         @request_object.send("#{line[:line_type]}_lines").build(attributes)
       end
       @request_object.save!
@@ -72,7 +73,16 @@ module RequestLogAnalyzer::Aggregator
         t.column(:request_id, :integer)
         t.column(:lineno, :integer)
         definition.captures.each do |capture|
-          t.column(capture[:name], column_type(capture))
+
+          # Add a field for every capture
+          t.column(capture[:name], column_type(capture[:type]))
+          
+          # If the capture provides other field as well, create them too
+          if capture[:provides].kind_of?(Hash)
+            capture[:provides].each do |field, field_type|
+              t.column(field, column_type(field_type))
+            end
+          end
         end
       end
       ActiveRecord::Migration.add_index("#{name}_lines", [:request_id])
@@ -86,6 +96,11 @@ module RequestLogAnalyzer::Aggregator
       class_name = "#{name}_line".camelize
       klass = Class.new(ActiveRecord::Base)
       klass.send(:belongs_to, :request)
+      
+      definition.captures.each do |capture|
+        klass.send(:serialize, capture[:name], Hash) if capture[:provides]
+      end
+      
       @orm_module.const_set(class_name, klass) unless @orm_module.const_defined?(class_name)
       @request_class.send(:has_many, "#{name}_lines".to_sym)
     end    
@@ -136,12 +151,22 @@ module RequestLogAnalyzer::Aggregator
     
     # Function to determine the column type for a field
     # TODO: make more robust / include in file-format definition
-    def column_type(capture)
-      case capture[:type]
-      when :sec;   :double
-      when :msec;  :double
-      when :float; :double
-      else         capture[:type]
+    def column_type(type_indicator)
+      case type_indicator
+      when :eval;      :text
+      when :text;      :text
+      when :string;    :string
+      when :sec;       :double
+      when :msec;      :double
+      when :duration;  :double
+      when :float;     :double
+      when :double;    :double
+      when :integer;   :integer
+      when :int;       :int
+      when :timestamp; :datetime
+      when :datetime;  :datetime
+      when :date;      :date
+      else             :string
       end
     end
   end

data/lib/request_log_analyzer/aggregator/echo.rb

@@ -1,5 +1,6 @@
 module RequestLogAnalyzer::Aggregator
 
+  # Echo Aggregator. Writes everything passed to it 
   class Echo < Base
     
     def prepare

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -1,5 +1,3 @@
-require File.dirname(__FILE__) + '/../tracker'
-
 module RequestLogAnalyzer::Aggregator
 
   class Summarizer < Base
@@ -11,6 +9,10 @@ module RequestLogAnalyzer::Aggregator
       def initialize
         @trackers = []
       end
+
+      def initialize_copy(other)
+        @trackers = other.trackers.dup
+      end
       
       def reset!
         @trackers = []
@@ -58,7 +60,7 @@ module RequestLogAnalyzer::Aggregator
     def prepare
       raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
       @trackers.each { |tracker| tracker.prepare }
-      end
+    end
     
     def aggregate(request)
       @trackers.each do |tracker|
@@ -86,23 +88,23 @@ module RequestLogAnalyzer::Aggregator
   
       output.with_style(:cell_separator => false) do 
         output.table({:width => 20}, {:font => :bold}) do |rows|
-          rows << ['Parsed lines:',   source.parsed_lines]
-          rows << ['Parsed request:', source.parsed_requests]
-          rows << ['Skipped lines:',  source.skipped_lines]
+          rows << ['Parsed lines:',    source.parsed_lines]
+          rows << ['Parsed requests:', source.parsed_requests]
+          rows << ['Skipped lines:',   source.skipped_lines]
         
-          rows <<  ["Warnings:", @warnings_encountered.map { |(key, value)| "#{key.inspect}: #{value}" }.join(', ')] if has_warnings?
+          rows << ["Warnings:", @warnings_encountered.map { |(key, value)| "#{key}: #{value}" }.join(', ')] if has_warnings?
         end
       end
       output << "\n"
     end
     
     def report_footer(output)
-      if has_serious_warnings?     
-        
+      if has_log_ordering_warnings?
         output.title("Parse warnings")
         
-        output.puts "Multiple warnings were encountered during parsing. Possibly, your logging "
-        output.puts "is not setup correctly. Visit this website for logging configuration tips:"
+        output.puts "Parseable lines were ancountered without a header line before it. It"
+        output.puts "could be that logging is not setup correctly for your application."
+        output.puts "Visit this website for logging configuration tips:"
         output.puts output.link("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging")
         output.puts
       end
@@ -112,8 +114,8 @@ module RequestLogAnalyzer::Aggregator
        @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
     end
     
-    def has_serious_warnings?
-      @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 10
+    def has_log_ordering_warnings?
+      @warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
     end
     
     def warning(type, message, lineno)

data/lib/request_log_analyzer/controller.rb

@@ -64,7 +64,7 @@ module RequestLogAnalyzer
       
       controller = Controller.new(RequestLogAnalyzer::Source::LogParser.new(file_format, options), options)
 
-      options[:assume_correct_order] = arguments[:assume_correct_order]
+      options[:parse_strategy] = arguments[:parse_strategy]
       
       # register filters        
       if arguments[:after] || arguments[:before]
@@ -160,6 +160,9 @@ module RequestLogAnalyzer
       @filters << filter.new(file_format, @options.merge(filter_options))
     end
     
+    # Push a request through the entire filterchain (@filters).
+    # <tt>request</tt> The request to filter.
+    # Returns the filtered request or nil.
     def filter_request(request)
       @filters.each do |filter| 
         request = filter.filter(request)
@@ -168,7 +171,10 @@ module RequestLogAnalyzer
       return request
     end
     
+    # Push a request to all the aggregators (@aggregators).
+    # <tt>request</tt> The request to push to the aggregators.    
     def aggregate_request(request)
+      return unless request
       @aggregators.each { |agg| agg.aggregate(request) }
     end
     
@@ -182,13 +188,11 @@ module RequestLogAnalyzer
     # 6. Finalize Source
     def run!
       
-      @filters.each { |filter| filter.prepare }
       @aggregators.each { |agg| agg.prepare }
       
       begin
         @source.each_request do |request|
-          request = filter_request(request)
-          aggregate_request(request) unless request.nil?
+          aggregate_request(filter_request(request))
         end
       rescue Interrupt => e
         handle_progress(:interrupted)

data/lib/request_log_analyzer/file_format/merb.rb

@@ -14,8 +14,9 @@ module RequestLogAnalyzer::FileFormat
     # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
     line_definition :params do |line|
       line.teaser = /Params/
-      line.regexp = /Params\:\ \{(.+)\}/
-      line.captures << { :name => :raw_hash, :type => :string}
+      line.regexp = /Params\:\ (\{.+\})/
+      line.captures << { :name => :params, :type => :eval, :provides => { 
+            :namespace => :string, :controller => :string, :action => :string, :format => :string } }
     end
 
     # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
@@ -29,11 +30,23 @@ module RequestLogAnalyzer::FileFormat
                     << { :name => :action_time,         :type => :duration }
     end
     
+    REQUEST_CATEGORIZER = Proc.new do |request| 
+      category = "#{request[:controller]}##{request[:action]}"
+      category = "#{request[:namespace]}::#{category}" if request[:namespace]
+      category = "#{category}.#{request[:format]}"     if request[:format]
+      category
+    end
     
     report do |analyze|
-      # FIX ME
+      analyze.timespan :line_type => :started
+      analyze.frequency :category => REQUEST_CATEGORIZER, :amount => 20, :title => "Top 20 by hits"
+      analyze.hourly_spread :line_type => :started
+      analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
+      # analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
+      # analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
+      # analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
     end
-    
+        
   end
 
 end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -76,8 +76,8 @@ module RequestLogAnalyzer::FileFormat
       analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
     
       analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
-      analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
-      analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+      analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+      analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
     
       analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
               :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20

data/lib/request_log_analyzer/file_format/rails_development.rb

@@ -1,28 +1,19 @@
 module RequestLogAnalyzer::FileFormat
   
-  class RailsDevelopment < Base
+  # The RailsDevelopment FileFormat is an extention to the default Rails file format. It includes
+  # all lines of the normal Rails file format, but parses SQL queries and partial rendering lines
+  # as well.
+  class RailsDevelopment < Rails
   
-    # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
-    line_definition :processing do |line|
-      line.header = true # this line is the first log line for a request 
-      line.teaser = /Processing /
-      line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
-      line.captures << { :name => :controller, :type  => :string } \
-                    << { :name => :action,     :type  => :string } \
-                    << { :name => :format,     :type  => :string } \
-                    << { :name => :ip,         :type  => :string } \
-                    << { :name => :timestamp,  :type  => :timestamp } \
-                    << { :name => :method,     :type  => :string }
-    end
-
-    # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.    
-    line_definition :cache_hit do |line|
-      line.regexp = /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/
+    #   Parameters: {"action"=>"demo", "controller"=>"page"}
+    line_definition :parameters do |line|
+      line.teaser = /Parameters/
+      line.regexp = /\s+Parameters:\s+(\{.*\})/
+      line.captures << { :name => :params, :type => :eval }
     end
   
     # Rendered layouts/_footer (2.9ms)
     line_definition :rendered do |line|
-      line.teaser = /Rendered /
       line.regexp = /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/
       line.captures << { :name => :render_file,     :type  => :string } \
                     << { :name => :render_duration, :type  => :duration, :unit => :msec }
@@ -30,90 +21,37 @@ module RequestLogAnalyzer::FileFormat
   
     #  User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) 
     line_definition :query_executed do |line|
-      line.regexp = /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?(.+) (?:\e\[0m)?/
+      line.regexp = /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/
       line.captures << { :name => :query_class,    :type  => :string } \
                     << { :name => :query_duration, :type  => :duration, :unit => :msec } \
-                    << { :name => :query_sql,      :type  => :string }
+                    << { :name => :query_sql,      :type  => :sql }
     end
 
     #  CACHE (0.0ms)   SELECT * FROM `users` WHERE (`users`.`id` = 0)   
     line_definition :query_cached do |line|
-      line.teaser = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?(.+) (?:\e\[0m)?/  
-      line.regexp = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?(.+) (?:\e\[0m)?/
+      line.regexp = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/
       line.captures << { :name => :cached_duration, :type  => :duration, :unit => :msec } \
-                    << { :name => :cached_sql,      :type  => :string }
+                    << { :name => :cached_sql,      :type  => :sql }
     end  
-  
-    # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
-    line_definition :failed do |line|
-      line.footer = true
-      line.regexp = /((?:[A-Z]\w+\:\:)*[A-Z]\w+) \((.*)\)(?: on line #(\d+) of .+)?\:(.*)/
-      line.captures << { :name => :error,       :type => :string } \
-                    << { :name => :message,     :type => :string } \
-                    << { :name => :line,        :type => :integer } \
-                    << { :name => :file,        :type => :string } \
-                    << { :name => :stack_trace, :type => :string }
-    end
-
 
-    # Rails < 2.1 completed line example
-    # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
-    RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
-
-    # Rails > 2.1 completed line example
-    # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
-    RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
-
-    # The completed line uses a kind of hack to ensure that both old style logs and new style logs 
-    # are both parsed by the same regular expression. The format in Rails 2.2 was slightly changed,
-    # but the line contains exactly the same information.
-    line_definition :completed do |line|
-    
-      line.footer = true
-      line.teaser = /Completed in /
-      line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
-    
-      line.captures << { :name => :duration, :type => :duration } \
-                    << { :name => :view,     :type => :duration } \
-                    << { :name => :db,       :type => :duration } \
-                    << { :name => :status,   :type => :integer } \
-                    << { :name => :url,      :type => :string } # Old variant 
-                  
-      line.captures << { :name => :duration, :type => :duration, :unit => :msec } \
-                    << { :name => :view,     :type => :duration, :unit => :msec } \
-                    << { :name => :db,       :type => :duration, :unit => :msec } \
-                    << { :name => :status,   :type => :integer} \
-                    << { :name => :url,      :type => :string }  # 2.2 variant 
-    end
-
-    REQUEST_CATEGORIZER = Proc.new do |request|
-      format = request[:format] || 'html'
-      "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-    end
-
-    report do |analyze|
-      analyze.timespan :line_type => :processing
-      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
-      analyze.frequency :method, :title => 'HTTP methods'
-      analyze.frequency :status, :title => 'HTTP statuses returned'
-      analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
-    
-      analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
-      analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
-      analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
-    
-      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
-              :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
-            
-      analyze.hourly_spread :line_type => :processing
-      analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
-    end
+    # Define the reporting for the additional parsed lines
+    report(:append) do |analyze|
+      
+      analyze.duration :render_duration, :category => :render_file, :multiple_per_request => true, 
+              :amount => 20, :title => 'Partial rendering duration'
+              
+      analyze.duration :query_duration, :category => :query_sql, :multiple_per_request => true, 
+              :amount => 20, :title => 'Query duration'
+              
+    end  
     
-    class Request < RequestLogAnalyzer::Request
-
-      def convert_timestamp(value, definition)
-        value.gsub(/[^0-9]/)[0...14].to_i unless value.nil?
+    # Add a converter method for the SQL fields the the Rails request class
+    class Request < Rails::Request
+      
+      # Sanitizes SQL queries so that they can be grouped
+      def convert_sql(sql, definition) 
+        sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
       end
-    end    
+    end
   end
 end

data/lib/request_log_analyzer/file_format.rb

@@ -80,8 +80,8 @@ module RequestLogAnalyzer::FileFormat
         
         # Copy the line and report definer from the parent class.
         subclass.class_eval do 
-          instance_variable_set(:@line_definer, superclass.line_definer)
-          instance_variable_set(:@report_definer, superclass.report_definer)
+          instance_variable_set(:@line_definer, superclass.line_definer.clone)
+          instance_variable_set(:@report_definer, superclass.report_definer.clone)
           class << self; attr_accessor :line_definer, :report_definer; end
         end        
         
@@ -95,8 +95,12 @@ module RequestLogAnalyzer::FileFormat
       @line_definer.send(name, &block)
     end
     
-    def create_request(*hashes)
-      self.class::Request.create(self, *hashes)
+    def request_class
+      self.class::Request
+    end
+    
+    def request(*hashes)
+      request_class.create(self, *hashes)
     end
     
     # Specifies multiple line definitions at once using a block

data/lib/request_log_analyzer/filter/anonymize.rb

@@ -7,9 +7,6 @@ module RequestLogAnalyzer::Filter
   # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
   class Anonymize < Base
    
-    def prepare
-    end
-
     def generate_random_ip
       "#{rand(256)}.#{rand(256)}.#{rand(256)}.#{rand(256)}"
     end

data/lib/request_log_analyzer/filter/field.rb

@@ -9,8 +9,13 @@ module RequestLogAnalyzer::Filter
    
     attr_reader :field, :value, :mode
    
+    def initialize(file_format, options = {})
+      super(file_format, options)
+      setup_filter
+    end
+   
     # Setup mode, field and value.
-    def prepare
+    def setup_filter
       @mode = (@options[:mode] || :accept).to_sym
       @field = @options[:field].to_sym