request-log-analyzer 1.3.4 → 1.3.6

data/lib/cli/database_console_init.rb

@@ -3,6 +3,7 @@ $:.unshift(File.dirname(__FILE__) + '/..')
 
 $database = RequestLogAnalyzer::Database.new(ENV['RLA_DBCONSOLE_DATABASE'])
 $database.load_database_schema!
+$database.register_default_orm_classes!
 
 require 'cli/tools'
 
@@ -39,4 +40,4 @@ end
 puts "request-log-analyzer database console"
 puts "-------------------------------------"
 puts "The following ActiveRecord classes are available:"
-puts $database.orm_classes.join(", ")
+puts $database.orm_classes.map { |k| k.name.split('::').last }.join(", ")

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -31,7 +31,7 @@ module RequestLogAnalyzer::Aggregator
     # This will create a record in the requests table and create a record for every line that has been parsed,
     # in which the captured values will be stored.
     def aggregate(request)
-      @request_object = database.request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      @request_object = RequestLogAnalyzer::Database::Request.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
       request.lines.each do |line|
         class_columns = database.get_class(line[:line_type]).column_names.reject { |column| ['id', 'source_id', 'request_id'].include?(column) }
         attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
@@ -45,23 +45,25 @@ module RequestLogAnalyzer::Aggregator
     
     # Finalizes the aggregator by closing the connection to the database
     def finalize
-      @request_count = database.request_class.count
+      @request_count = RequestLogAnalyzer::Database::Request.count
       database.disconnect
       database.remove_orm_classes!
     end
     
     # Records w warining in the warnings table.
     def warning(type, message, lineno)
-      database.warning_class.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+      RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
     end
     
     # Records source changes in the sources table
     def source_change(change, filename)
-      case change
-      when :started
-        @sources[filename] = database.source_class.create!(:filename => filename)
-      when :finished
-        @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
+      if File.exist?(filename)
+        case change
+        when :started
+          @sources[filename] = RequestLogAnalyzer::Database::Source.create!(:filename => filename)
+        when :finished
+          @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
+        end
       end
     end
     

data/lib/request_log_analyzer/aggregator.rb

@@ -8,16 +8,12 @@ module RequestLogAnalyzer::Aggregator
   # every aggregator should comply (by simply subclassing this class).
   class Base
     
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    attr_reader :options
-    attr_reader :source
+    attr_reader :options, :source
 
     # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
     # It will include the specific file format module.
     def initialize(source, options = {})
       @source = source
-      self.register_file_format(source.file_format)
       @options = options
     end
 

data/lib/request_log_analyzer/controller.rb

@@ -17,14 +17,7 @@ module RequestLogAnalyzer
   # from several logrotated log files.
   class Controller
 
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    attr_reader :aggregators
-    attr_reader :filters
-    attr_reader :log_parser
-    attr_reader :source
-    attr_reader :output
-    attr_reader :options
+    attr_reader :source, :filters, :aggregators, :output, :options
 
     # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
@@ -35,8 +28,8 @@ module RequestLogAnalyzer
       # Database command line options
       options[:database]       = arguments[:database] if arguments[:database]
       options[:reset_database] = arguments[:reset_database]
-      options[:debug]    = arguments[:debug]
-      options[:dump]     = arguments[:dump]
+      options[:debug]          = arguments[:debug]
+      options[:dump]           = arguments[:dump]
       options[:parse_strategy] = arguments[:parse_strategy]
       options[:no_progress]    = arguments[:no_progress]
       
@@ -125,17 +118,13 @@ module RequestLogAnalyzer
       @filters     = []
       @output      = options[:output]
       
-      # Requester format through RequestLogAnalyzer::FileFormat and construct the parser
-      register_file_format(@source.file_format) 
-      
-      # Pass all warnings to every aggregator so they can do something useful with them.
-      @source.warning = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } } if @source
-
-      # Handle progress messagess
-      @source.progress = lambda { |message, value| handle_progress(message, value) } if @source && !options[:no_progress]
+      # Register the request format for this session after checking its validity
+      raise "Invalid file format!" unless @source.file_format.valid?
       
-      # Handle source change messages
-      @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) } if @source
+      # Install event handlers for wrnings, progress updates and source changes
+      @source.warning        = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } }
+      @source.progress       = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
+      @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
     end
     
     # Progress function.
@@ -176,7 +165,7 @@ module RequestLogAnalyzer
     # Adds a request filter to the controller.
     def add_filter(filter, filter_options = {})
       filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
-      @filters << filter.new(file_format, @options.merge(filter_options))
+      @filters << filter.new(source.file_format, @options.merge(filter_options))
     end
     
     # Push a request through the entire filterchain (@filters).

data/lib/request_log_analyzer/database/base.rb

@@ -13,8 +13,9 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
   def line_type
     self.class.name.underscore.gsub(/_line$/, '').to_sym
   end
-
-  cattr_accessor :database, :line_definition
+  
+  class_inheritable_accessor :line_definition
+  cattr_accessor :database
 
   def self.subclass_from_line_definition(definition)
     klass = Class.new(RequestLogAnalyzer::Database::Base)
@@ -31,8 +32,8 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
       klass.send(:serialize, capture[:name], Hash)
     end
 
-    database.request_class.has_many  "#{definition.name}_lines".to_sym
-    database.source_class.has_many   "#{definition.name}_lines".to_sym
+    RequestLogAnalyzer::Database::Request.has_many  "#{definition.name}_lines".to_sym
+    RequestLogAnalyzer::Database::Source.has_many   "#{definition.name}_lines".to_sym
 
     return klass
   end
@@ -45,12 +46,12 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
 
     if klass.column_names.include?('request_id')
       klass.belongs_to :request
-      database.request_class.has_many table.to_sym
+      RequestLogAnalyzer::Database::Request.has_many table.to_sym
     end
     
     if klass.column_names.include?('source_id')
       klass.belongs_to :source
-      database.source_class.has_many table.to_sym
+      RequestLogAnalyzer::Database::Source.has_many table.to_sym
     end
     
     return klass

data/lib/request_log_analyzer/database/request.rb

@@ -0,0 +1,22 @@
+class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
+  
+  # Returns an array of all the Line objects of this request in the correct order.
+  def lines
+    @lines ||= begin
+      lines = []
+      self.class.reflections.each { |r, d| lines += self.send(r).all }
+      lines.sort
+    end
+  end
+  
+  # Creates the table to store requests in.
+  def self.create_table!
+    unless database.connection.table_exists?(:requests)
+      database.connection.create_table(:requests) do |t|
+        t.column :first_lineno, :integer
+        t.column :last_lineno,  :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/source.rb

@@ -0,0 +1,13 @@
+class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
+  
+  def self.create_table!
+    unless database.connection.table_exists?(:sources)
+      database.connection.create_table(:sources) do |t|
+        t.column :filename, :string
+        t.column :mtime,    :datetime
+        t.column :filesize, :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/warning.rb

@@ -0,0 +1,14 @@
+class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
+
+  def self.create_table!
+    unless database.connection.table_exists?(:warnings)
+      database.connection.create_table(:warnings) do |t|
+        t.column  :warning_type, :string, :limit => 30, :null => false
+        t.column  :message, :string
+        t.column  :source_id, :integer
+        t.column  :lineno, :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database.rb

@@ -10,7 +10,7 @@ class RequestLogAnalyzer::Database
   include RequestLogAnalyzer::Database::Connection
 
   attr_accessor :file_format
-  attr_reader :request_class, :warning_class, :source_class, :line_classes
+  attr_reader :line_classes
   
   def initialize(connection_identifier = nil)
     @line_classes = []
@@ -24,97 +24,17 @@ class RequestLogAnalyzer::Database
     Object.const_get("#{line_type}_line".camelize)
   end
   
-  # Returns the Request ORM class for the current database.
-  #
-  # It will create the class if not previously done so. The class will
-  # include a create_table! method the migrate the database.
-  def request_class
-    @request_class ||= begin
-      klass = Class.new(RequestLogAnalyzer::Database::Base) do
-        
-        def lines
-          @lines ||= begin
-            lines = []
-            self.class.reflections.each { |r, d| lines += self.send(r).all }
-            lines.sort
-          end
-        end
-        
-        # Creates the requests table
-        def self.create_table!
-          unless database.connection.table_exists?(:requests)
-            database.connection.create_table(:requests) do |t|
-              t.column :first_lineno, :integer
-              t.column :last_lineno,  :integer
-            end
-          end
-        end
-      end
-      
-      Object.const_set('Request', klass) 
-      Object.const_get('Request')
-    end
-  end
-  
-  # Returns the Source ORM class for the current database.
-  #
-  # It will create the class if not previously done so. The class will
-  # include a create_table! method the migrate the database.  
-  def source_class
-    @source_class ||= begin
-      klass = Class.new(RequestLogAnalyzer::Database::Base) do
-        
-        # Creates the sources table
-        def self.create_table!
-          unless database.connection.table_exists?(:sources)
-            database.connection.create_table(:sources) do |t|
-              t.column :filename, :string
-              t.column :mtime,    :datetime
-              t.column :filesize, :integer
-            end
-          end
-        end
-      end
-      
-      Object.const_set('Source', klass)
-      Object.const_get('Source')
-    end
-  end
-  
-  
-  # Returns the Warning ORM class for the current database.
-  #
-  # It will create the class if not previously done so. The class will
-  # include a create_table! method the migrate the database.  
-  def warning_class
-    @warning_class ||= begin
-      klass = Class.new(RequestLogAnalyzer::Database::Base) do
-        
-        # Creates the warnings table
-        def self.create_table!
-          unless database.connection.table_exists?(:warnings)
-            database.connection.create_table(:warnings) do |t|
-              t.column  :warning_type, :string, :limit => 30, :null => false
-              t.column  :message, :string
-              t.column  :source_id, :integer
-              t.column  :lineno, :integer
-            end
-          end
-        end
-      end
-      
-      Object.const_set('Warning', klass)
-      Object.const_get('Warning')
-    end
+  def default_classes
+    [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
   end
   
   # Loads the ORM classes by inspecting the tables in the current database
   def load_database_schema!
     connection.tables.map do |table|
       case table.to_sym
-      when :warnings then warning_class
-      when :sources  then source_class
-      when :requests then request_class
+      when :warnings then RequestLogAnalyzer::Database::Warning
+      when :sources  then RequestLogAnalyzer::Database::Source
+      when :requests then RequestLogAnalyzer::Database::Request
       else load_activerecord_class(table)
       end
     end
@@ -122,7 +42,7 @@ class RequestLogAnalyzer::Database
   
   # Returns an array of all the ActiveRecord-bases ORM classes for this database
   def orm_classes
-    [warning_class, request_class, source_class] + line_classes
+    default_classes + line_classes
   end
   
   # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
@@ -146,8 +66,6 @@ class RequestLogAnalyzer::Database
 
   def fileformat_classes
     raise "No file_format provided!" unless file_format
-    
-    default_classes = [request_class, source_class, warning_class]
     line_classes    = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
     return default_classes + line_classes
   end
@@ -164,12 +82,19 @@ class RequestLogAnalyzer::Database
     remove_orm_classes!
   end
   
+  # Registers the default ORM classes in the default namespace
+  def register_default_orm_classes!
+    Object.const_set('Request', RequestLogAnalyzer::Database::Request)
+    Object.const_set('Source',  RequestLogAnalyzer::Database::Source)
+    Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
+  end
+  
   # Unregisters every ORM class constant
   def remove_orm_classes!
     orm_classes.each do |klass|
       if klass.respond_to?(:name) && !klass.name.blank?
-#        klass_base_name = klass.name.split('::').last
-        Object.send(:remove_const, klass.name) if Object.const_defined?(klass.name)
+        klass_name = klass.name.split('::').last
+        Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name)
       end
     end
   end

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -9,7 +9,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :access do |line|
       line.header = true
       line.footer = true
-      line.regexp = /^([^\ ]+) ([^\ ]+) \[([^\]]{26})\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
+      line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
       line.captures << { :name => :bucket_owner,    :type => :string } << 
                        { :name => :bucket,          :type => :string } << 
                        { :name => :timestamp,       :type => :timestamp } <<
@@ -21,8 +21,8 @@ module RequestLogAnalyzer::FileFormat
                        { :name => :request_uri,     :type => :string } <<
                        { :name => :http_status,     :type => :integer } <<
                        { :name => :error_code,      :type => :nillable_string } <<
-                       { :name => :bytes_sent,      :type => :integer } <<
-                       { :name => :object_size,     :type => :integer } <<
+                       { :name => :bytes_sent,      :type => :traffic,  :unit => :byte } <<
+                       { :name => :object_size,     :type => :traffic,  :unit => :byte } <<
                        { :name => :total_time,      :type => :duration, :unit => :msec } <<
                        { :name => :turnaround_time, :type => :duration, :unit => :msec } <<
                        { :name => :referer,         :type => :referer } <<
@@ -34,7 +34,8 @@ module RequestLogAnalyzer::FileFormat
       analyze.hourly_spread
 
       analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Most popular items"
-      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Duration"
+      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Request duration"
+      analyze.traffic  :traffic => :bytes_sent,  :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Traffic"
       analyze.frequency :category => :http_status, :title => 'HTTP status codes'
       analyze.frequency :category => :error_code, :title => 'Error codes'
     end
@@ -47,8 +48,7 @@ module RequestLogAnalyzer::FileFormat
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
-        d = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})/.match(value).captures
-        "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
       
       # Make sure that the string '-' is parsed as a nil value.

data/lib/request_log_analyzer/file_format/apache.rb

@@ -17,26 +17,30 @@ module RequestLogAnalyzer::FileFormat
     # A hash of predefined Apache log formats
     LOG_FORMAT_DEFAULTS = {
       :common   => '%h %l %u %t "%r" %>s %b',
-      :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
+      :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
+      :rack     => '%h %l %u %t "%r" %>s %b %T',
+      :referer  => '%{Referer}i -> %U',
+      :agent    => '%{User-agent}i'
     }
-
+    
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
       '%' => { :regexp => '%', :captures => [] },
       'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
       'a' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :remote_ip, :type => :string}] },
-      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :integer}] },
+      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
       'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
-      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :msec}] },
+      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
       'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
-      'T' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
-      't' => { :regexp => '\[([^\]]{26})\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
+      'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
+      't' => { :regexp => '\[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
       's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
       'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
-      'r' => { :regexp => '([A-Z]+) ([^\s]+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
+      'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
+      'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
                        {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
-      'i' => { 'Referer'    => { :regexp => '([^\s]+)', :captures => [{:name => :referer, :type => :nillable_string}] },
-               'User-agent' => { :regexp => '(.*)',     :captures => [{:name => :user_agent, :type => :user_agent}] }
+      'i' => { 'Referer'    => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
+               'User-agent' => { :regexp => '(.*)',  :captures => [{:name => :user_agent, :type => :user_agent}] }
              }
     }
 
@@ -88,14 +92,13 @@ module RequestLogAnalyzer::FileFormat
 
       analyze.frequency :category => :http_method, :amount => 20, :title => "HTTP methods"  if line_definition.captures?(:http_method)
       analyze.frequency :category => :http_status, :amount => 20, :title => "HTTP statuses" if line_definition.captures?(:http_status)
-      analyze.frequency :category => :path, :amount => 20, :title => "Most popular URIs"    if line_definition.captures?(:path)
+      analyze.frequency :category => lambda { |r| r.category }, :amount => 20, :title => "Most popular URIs"    if line_definition.captures?(:path)
 
       analyze.frequency :category => :user_agent, :amount => 20, :title => "User agents"    if line_definition.captures?(:user_agent)
       analyze.frequency :category => :referer,    :amount => 20, :title => "Referers"       if line_definition.captures?(:referer)
 
-      if line_definition.captures?(:path) && line_definition.captures?(:duration)
-        analyze.duration :duration => :duration, :category => :path , :title => 'Request duration'
-      end
+      analyze.duration :duration => :duration,  :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
+      analyze.traffic  :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic'          if line_definition.captures?(:bytes_sent)
 
       return analyze.trackers
     end
@@ -103,14 +106,17 @@ module RequestLogAnalyzer::FileFormat
     # Define a custom Request class for the Apache file format to speed up timestamp handling.
     class Request < RequestLogAnalyzer::Request
       
+      def category
+        first(:path)
+      end
+      
       MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
                 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
       
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
-        d = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})/.match(value).captures
-        "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
       
       # This function can be overridden to rewrite the path for better categorization in the

data/lib/request_log_analyzer/file_format/merb.rb

@@ -1,11 +1,14 @@
 module RequestLogAnalyzer::FileFormat
   
+  # The Merb file format parses the request header with the timestamp, the params line
+  # with the most important request information and the durations line which contains
+  # the different request durations that can be used for analysis.
   class Merb < Base
 
     # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
     line_definition :started do |line|
       line.header = true
-      line.teaser = /Started/
+      # line.teaser = /Started/
       line.regexp = /Started request handling\:\ (.+)/
       line.captures << { :name => :timestamp, :type => :timestamp }
     end    
@@ -13,7 +16,7 @@ module RequestLogAnalyzer::FileFormat
     # ~ Params: {"action"=>"create", "controller"=>"session"}
     # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
     line_definition :params do |line|
-      line.teaser = /Params/
+      # line.teaser = /Params/
       line.regexp = /Params\:\ (\{.+\})/
       line.captures << { :name => :params, :type => :eval, :provides => { 
             :namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
@@ -36,15 +39,28 @@ module RequestLogAnalyzer::FileFormat
     end
     
     report do |analyze|
-      analyze.timespan :line_type => :started
+      
+      analyze.timespan
+      analyze.hourly_spread
+
       analyze.frequency :category => REQUEST_CATEGORIZER, :amount => 20, :title => "Top 20 by hits"
-      analyze.hourly_spread :line_type => :started
       analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
+      
       # analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
       # analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
       # analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
     end
-        
+
+    class Request < RequestLogAnalyzer::Request
+      
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+      
+      # Speed up timestamp conversion
+      def convert_timestamp(value, definition)
+        "#{value[26,4]}#{MONTHS[value[4,3]]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
+      end
+    end
   end
 
 end

data/lib/request_log_analyzer/file_format/rack.rb

@@ -0,0 +1,11 @@
+module RequestLogAnalyzer::FileFormat
+  
+  class Rack < Apache
+    
+    def self.create(*args)
+      super(:rack, *args)
+    end
+    
+  end
+  
+end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -5,11 +5,11 @@ module RequestLogAnalyzer::FileFormat
     # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
     line_definition :processing do |line|
       line.header = true # this line is the first log line for a request 
-      line.teaser = /Processing /
+      # line.teaser = /Processing /
       line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
       line.captures << { :name => :controller, :type  => :string } \
                     << { :name => :action,     :type  => :string } \
-                    << { :name => :format,     :type  => :format } \
+                    << { :name => :format,     :type  => :string, :default => 'html' } \
                     << { :name => :ip,         :type  => :string } \
                     << { :name => :timestamp,  :type  => :timestamp } \
                     << { :name => :method,     :type  => :string }
@@ -46,7 +46,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :completed do |line|
     
       line.footer = true
-      line.teaser = /Completed in /
+      # line.teaser = /Completed in /
       line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
     
       line.captures << { :name => :duration, :type => :duration, :unit => :sec } \
@@ -62,15 +62,15 @@ module RequestLogAnalyzer::FileFormat
                     << { :name => :url,      :type => :string }  # 2.2 variant 
     end
 
-
-
     REQUEST_CATEGORIZER = Proc.new do |request|
       "#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
     end
 
     report do |analyze|
-      analyze.timespan :line_type => :processing
-      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
+      analyze.timespan
+      analyze.hourly_spread
+            
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20
       analyze.frequency :method, :title => 'HTTP methods'
       analyze.frequency :status, :title => 'HTTP statuses returned'
       analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
@@ -82,23 +82,17 @@ module RequestLogAnalyzer::FileFormat
       analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
               :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
             
-      analyze.hourly_spread :line_type => :processing
       analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
     end
 
     # Define a custom Request class for the Rails file format to speed up timestamp handling
     # and to ensure that a format is always set.
     class Request < RequestLogAnalyzer::Request
-
+    
       # Do not use DateTime.parse
       def convert_timestamp(value, definition)
         value.gsub(/[^0-9]/, '')[0...14].to_i unless value.nil?
       end
-      
-      # Set 'html' as default format for a request
-      def convert_format(value, definition)
-        value || 'html'
-      end
     end
 
   end