wvanbergen-request-log-analyzer 1.3.5 → 1.3.6

data/lib/cli/database_console_init.rb

@@ -3,6 +3,7 @@ $:.unshift(File.dirname(__FILE__) + '/..')
 
 $database = RequestLogAnalyzer::Database.new(ENV['RLA_DBCONSOLE_DATABASE'])
 $database.load_database_schema!
+$database.register_default_orm_classes!
 
 require 'cli/tools'
 
@@ -39,4 +40,4 @@ end
 puts "request-log-analyzer database console"
 puts "-------------------------------------"
 puts "The following ActiveRecord classes are available:"
-puts $database.orm_classes.join(", ")
+puts $database.orm_classes.map { |k| k.name.split('::').last }.join(", ")

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
   
   # The current version of request-log-analyzer.
   # This will be diplayed in output reports etc.
-  VERSION = "1.3.5"
+  VERSION = "1.3.6"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/lib/request_log_analyzer/aggregator.rb

@@ -8,16 +8,12 @@ module RequestLogAnalyzer::Aggregator
   # every aggregator should comply (by simply subclassing this class).
   class Base
     
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    attr_reader :options
-    attr_reader :source
+    attr_reader :options, :source
 
     # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
     # It will include the specific file format module.
     def initialize(source, options = {})
       @source = source
-      self.register_file_format(source.file_format)
       @options = options
     end
 

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -31,7 +31,7 @@ module RequestLogAnalyzer::Aggregator
     # This will create a record in the requests table and create a record for every line that has been parsed,
     # in which the captured values will be stored.
     def aggregate(request)
-      @request_object = database.request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      @request_object = RequestLogAnalyzer::Database::Request.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
       request.lines.each do |line|
         class_columns = database.get_class(line[:line_type]).column_names.reject { |column| ['id', 'source_id', 'request_id'].include?(column) }
         attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
@@ -45,14 +45,14 @@ module RequestLogAnalyzer::Aggregator
     
     # Finalizes the aggregator by closing the connection to the database
     def finalize
-      @request_count = database.request_class.count
+      @request_count = RequestLogAnalyzer::Database::Request.count
       database.disconnect
       database.remove_orm_classes!
     end
     
     # Records w warining in the warnings table.
     def warning(type, message, lineno)
-      database.warning_class.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+      RequestLogAnalyzer::Database::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
     end
     
     # Records source changes in the sources table
@@ -60,8 +60,7 @@ module RequestLogAnalyzer::Aggregator
       if File.exist?(filename)
         case change
         when :started
-          p database.source_class
-          @sources[filename] = database.source_class.create!(:filename => filename)
+          @sources[filename] = RequestLogAnalyzer::Database::Source.create!(:filename => filename)
         when :finished
           @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
         end

data/lib/request_log_analyzer/controller.rb

@@ -17,14 +17,7 @@ module RequestLogAnalyzer
   # from several logrotated log files.
   class Controller
 
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    attr_reader :aggregators
-    attr_reader :filters
-    attr_reader :log_parser
-    attr_reader :source
-    attr_reader :output
-    attr_reader :options
+    attr_reader :source, :filters, :aggregators, :output, :options
 
     # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
@@ -35,8 +28,8 @@ module RequestLogAnalyzer
       # Database command line options
       options[:database]       = arguments[:database] if arguments[:database]
       options[:reset_database] = arguments[:reset_database]
-      options[:debug]    = arguments[:debug]
-      options[:dump]     = arguments[:dump]
+      options[:debug]          = arguments[:debug]
+      options[:dump]           = arguments[:dump]
       options[:parse_strategy] = arguments[:parse_strategy]
       options[:no_progress]    = arguments[:no_progress]
       
@@ -125,17 +118,13 @@ module RequestLogAnalyzer
       @filters     = []
       @output      = options[:output]
       
-      # Requester format through RequestLogAnalyzer::FileFormat and construct the parser
-      register_file_format(@source.file_format) 
-      
-      # Pass all warnings to every aggregator so they can do something useful with them.
-      @source.warning = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } } if @source
-
-      # Handle progress messagess
-      @source.progress = lambda { |message, value| handle_progress(message, value) } if @source && !options[:no_progress]
+      # Register the request format for this session after checking its validity
+      raise "Invalid file format!" unless @source.file_format.valid?
       
-      # Handle source change messages
-      @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) } if @source
+      # Install event handlers for wrnings, progress updates and source changes
+      @source.warning        = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } }
+      @source.progress       = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
+      @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
     end
     
     # Progress function.
@@ -176,7 +165,7 @@ module RequestLogAnalyzer
     # Adds a request filter to the controller.
     def add_filter(filter, filter_options = {})
       filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
-      @filters << filter.new(file_format, @options.merge(filter_options))
+      @filters << filter.new(source.file_format, @options.merge(filter_options))
     end
     
     # Push a request through the entire filterchain (@filters).

data/lib/request_log_analyzer/database.rb

@@ -10,7 +10,7 @@ class RequestLogAnalyzer::Database
   include RequestLogAnalyzer::Database::Connection
 
   attr_accessor :file_format
-  attr_reader :request_class, :warning_class, :source_class, :line_classes
+  attr_reader :line_classes
   
   def initialize(connection_identifier = nil)
     @line_classes = []
@@ -24,97 +24,17 @@ class RequestLogAnalyzer::Database
     Object.const_get("#{line_type}_line".camelize)
   end
   
-  # Returns the Request ORM class for the current database.
-  #
-  # It will create the class if not previously done so. The class will
-  # include a create_table! method the migrate the database.
-  def request_class
-    @request_class ||= begin
-      klass = Class.new(RequestLogAnalyzer::Database::Base) do
-        
-        def lines
-          @lines ||= begin
-            lines = []
-            self.class.reflections.each { |r, d| lines += self.send(r).all }
-            lines.sort
-          end
-        end
-        
-        # Creates the requests table
-        def self.create_table!
-          unless database.connection.table_exists?(:requests)
-            database.connection.create_table(:requests) do |t|
-              t.column :first_lineno, :integer
-              t.column :last_lineno,  :integer
-            end
-          end
-        end
-      end
-      
-      Object.const_set('Request', klass) 
-      Object.const_get('Request')
-    end
-  end
-  
-  # Returns the Source ORM class for the current database.
-  #
-  # It will create the class if not previously done so. The class will
-  # include a create_table! method the migrate the database.  
-  def source_class
-    @source_class ||= begin
-      klass = Class.new(RequestLogAnalyzer::Database::Base) do
-        
-        # Creates the sources table
-        def self.create_table!
-          unless database.connection.table_exists?(:sources)
-            database.connection.create_table(:sources) do |t|
-              t.column :filename, :string
-              t.column :mtime,    :datetime
-              t.column :filesize, :integer
-            end
-          end
-        end
-      end
-      
-      Object.const_set('Source', klass)
-      Object.const_get('Source')
-    end
-  end
-  
-  
-  # Returns the Warning ORM class for the current database.
-  #
-  # It will create the class if not previously done so. The class will
-  # include a create_table! method the migrate the database.  
-  def warning_class
-    @warning_class ||= begin
-      klass = Class.new(RequestLogAnalyzer::Database::Base) do
-        
-        # Creates the warnings table
-        def self.create_table!
-          unless database.connection.table_exists?(:warnings)
-            database.connection.create_table(:warnings) do |t|
-              t.column  :warning_type, :string, :limit => 30, :null => false
-              t.column  :message, :string
-              t.column  :source_id, :integer
-              t.column  :lineno, :integer
-            end
-          end
-        end
-      end
-      
-      Object.const_set('Warning', klass)
-      Object.const_get('Warning')
-    end
+  def default_classes
+    [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
   end
   
   # Loads the ORM classes by inspecting the tables in the current database
   def load_database_schema!
     connection.tables.map do |table|
       case table.to_sym
-      when :warnings then warning_class
-      when :sources  then source_class
-      when :requests then request_class
+      when :warnings then RequestLogAnalyzer::Database::Warning
+      when :sources  then RequestLogAnalyzer::Database::Source
+      when :requests then RequestLogAnalyzer::Database::Request
       else load_activerecord_class(table)
       end
     end
@@ -122,7 +42,7 @@ class RequestLogAnalyzer::Database
   
   # Returns an array of all the ActiveRecord-bases ORM classes for this database
   def orm_classes
-    [warning_class, request_class, source_class] + line_classes
+    default_classes + line_classes
   end
   
   # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
@@ -146,8 +66,6 @@ class RequestLogAnalyzer::Database
 
   def fileformat_classes
     raise "No file_format provided!" unless file_format
-    
-    default_classes = [request_class, source_class, warning_class]
     line_classes    = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
     return default_classes + line_classes
   end
@@ -164,12 +82,19 @@ class RequestLogAnalyzer::Database
     remove_orm_classes!
   end
   
+  # Registers the default ORM classes in the default namespace
+  def register_default_orm_classes!
+    Object.const_set('Request', RequestLogAnalyzer::Database::Request)
+    Object.const_set('Source',  RequestLogAnalyzer::Database::Source)
+    Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
+  end
+  
   # Unregisters every ORM class constant
   def remove_orm_classes!
     orm_classes.each do |klass|
       if klass.respond_to?(:name) && !klass.name.blank?
-#        klass_base_name = klass.name.split('::').last
-        Object.send(:remove_const, klass.name) if Object.const_defined?(klass.name)
+        klass_name = klass.name.split('::').last
+        Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name)
       end
     end
   end

data/lib/request_log_analyzer/database/base.rb

@@ -32,8 +32,8 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
       klass.send(:serialize, capture[:name], Hash)
     end
 
-    database.request_class.has_many  "#{definition.name}_lines".to_sym
-    database.source_class.has_many   "#{definition.name}_lines".to_sym
+    RequestLogAnalyzer::Database::Request.has_many  "#{definition.name}_lines".to_sym
+    RequestLogAnalyzer::Database::Source.has_many   "#{definition.name}_lines".to_sym
 
     return klass
   end
@@ -46,12 +46,12 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
 
     if klass.column_names.include?('request_id')
       klass.belongs_to :request
-      database.request_class.has_many table.to_sym
+      RequestLogAnalyzer::Database::Request.has_many table.to_sym
     end
     
     if klass.column_names.include?('source_id')
       klass.belongs_to :source
-      database.source_class.has_many table.to_sym
+      RequestLogAnalyzer::Database::Source.has_many table.to_sym
     end
     
     return klass

data/lib/request_log_analyzer/database/request.rb

@@ -0,0 +1,22 @@
+class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
+  
+  # Returns an array of all the Line objects of this request in the correct order.
+  def lines
+    @lines ||= begin
+      lines = []
+      self.class.reflections.each { |r, d| lines += self.send(r).all }
+      lines.sort
+    end
+  end
+  
+  # Creates the table to store requests in.
+  def self.create_table!
+    unless database.connection.table_exists?(:requests)
+      database.connection.create_table(:requests) do |t|
+        t.column :first_lineno, :integer
+        t.column :last_lineno,  :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/source.rb

@@ -0,0 +1,13 @@
+class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
+  
+  def self.create_table!
+    unless database.connection.table_exists?(:sources)
+      database.connection.create_table(:sources) do |t|
+        t.column :filename, :string
+        t.column :mtime,    :datetime
+        t.column :filesize, :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/database/warning.rb

@@ -0,0 +1,14 @@
+class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base
+
+  def self.create_table!
+    unless database.connection.table_exists?(:warnings)
+      database.connection.create_table(:warnings) do |t|
+        t.column  :warning_type, :string, :limit => 30, :null => false
+        t.column  :message, :string
+        t.column  :source_id, :integer
+        t.column  :lineno, :integer
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/file_format.rb

@@ -42,19 +42,7 @@ module RequestLogAnalyzer::FileFormat
     raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
     
     @current_file_format = klass.create(*args) # return an instance of the class
-  end  
-  
-  # Makes classes aware of a file format by registering the file_format variable
-  module Awareness
-    
-    def self.included(base)
-      base.send(:attr_reader, :file_format)
-    end
-    
-    def register_file_format(format)
-      @file_format = format
-    end
-  end  
+  end
 
   # Base class for all log file format definitions. This class provides functions for subclasses to 
   # define their LineDefinitions and to define a summary report.

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -48,8 +48,7 @@ module RequestLogAnalyzer::FileFormat
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
-        d = /^(\d{2})\/([A-Za-z]{3})\/(\d{4}).(\d{2}):(\d{2}):(\d{2})/.match(value).captures
-        "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
       
       # Make sure that the string '-' is parsed as a nil value.

data/lib/request_log_analyzer/file_format/apache.rb

@@ -92,18 +92,13 @@ module RequestLogAnalyzer::FileFormat
 
       analyze.frequency :category => :http_method, :amount => 20, :title => "HTTP methods"  if line_definition.captures?(:http_method)
       analyze.frequency :category => :http_status, :amount => 20, :title => "HTTP statuses" if line_definition.captures?(:http_status)
-      analyze.frequency :category => :path, :amount => 20, :title => "Most popular URIs"    if line_definition.captures?(:path)
+      analyze.frequency :category => lambda { |r| r.category }, :amount => 20, :title => "Most popular URIs"    if line_definition.captures?(:path)
 
       analyze.frequency :category => :user_agent, :amount => 20, :title => "User agents"    if line_definition.captures?(:user_agent)
       analyze.frequency :category => :referer,    :amount => 20, :title => "Referers"       if line_definition.captures?(:referer)
 
-      if line_definition.captures?(:path) && line_definition.captures?(:duration)
-        analyze.duration :duration => :duration, :category => :path , :title => 'Request duration'
-      end
-
-      if line_definition.captures?(:path) && line_definition.captures?(:bytes_sent)
-        analyze.traffic :traffic => :bytes_sent, :category => :path , :title => 'Traffic'
-      end
+      analyze.duration :duration => :duration,  :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
+      analyze.traffic  :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic'          if line_definition.captures?(:bytes_sent)
 
       return analyze.trackers
     end
@@ -111,14 +106,17 @@ module RequestLogAnalyzer::FileFormat
     # Define a custom Request class for the Apache file format to speed up timestamp handling.
     class Request < RequestLogAnalyzer::Request
       
+      def category
+        first(:path)
+      end
+      
       MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
                 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
       
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
-        d = /^(\d{2})\/([A-Za-z]{3})\/(\d{4}).(\d{2}):(\d{2}):(\d{2})/.match(value).captures
-        "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
       
       # This function can be overridden to rewrite the path for better categorization in the

data/lib/request_log_analyzer/file_format/merb.rb

@@ -1,11 +1,14 @@
 module RequestLogAnalyzer::FileFormat
   
+  # The Merb file format parses the request header with the timestamp, the params line
+  # with the most important request information and the durations line which contains
+  # the different request durations that can be used for analysis.
   class Merb < Base
 
     # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
     line_definition :started do |line|
       line.header = true
-      line.teaser = /Started/
+      # line.teaser = /Started/
       line.regexp = /Started request handling\:\ (.+)/
       line.captures << { :name => :timestamp, :type => :timestamp }
     end    
@@ -13,7 +16,7 @@ module RequestLogAnalyzer::FileFormat
     # ~ Params: {"action"=>"create", "controller"=>"session"}
     # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
     line_definition :params do |line|
-      line.teaser = /Params/
+      # line.teaser = /Params/
       line.regexp = /Params\:\ (\{.+\})/
       line.captures << { :name => :params, :type => :eval, :provides => { 
             :namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
@@ -36,15 +39,28 @@ module RequestLogAnalyzer::FileFormat
     end
     
     report do |analyze|
-      analyze.timespan :line_type => :started
+      
+      analyze.timespan
+      analyze.hourly_spread
+
       analyze.frequency :category => REQUEST_CATEGORIZER, :amount => 20, :title => "Top 20 by hits"
-      analyze.hourly_spread :line_type => :started
       analyze.duration :dispatch_time, :category => REQUEST_CATEGORIZER, :title => 'Request dispatch duration'
+      
       # analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration'
       # analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration'
       # analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration'
     end
-        
+
+    class Request < RequestLogAnalyzer::Request
+      
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+      
+      # Speed up timestamp conversion
+      def convert_timestamp(value, definition)
+        "#{value[26,4]}#{MONTHS[value[4,3]]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
+      end
+    end
   end
 
 end