request-log-analyzer 1.3.7 → 1.4.0

data/lib/request_log_analyzer/database.rb

@@ -5,13 +5,13 @@ class RequestLogAnalyzer::Database
 
   def self.const_missing(const) # :nodoc:
     RequestLogAnalyzer::load_default_class_file(self, const)
-  end  
+  end
 
   include RequestLogAnalyzer::Database::Connection
 
   attr_accessor :file_format
   attr_reader :line_classes
-  
+
   def initialize(connection_identifier = nil)
     @line_classes = []
     RequestLogAnalyzer::Database::Base.database = self
@@ -23,11 +23,11 @@ class RequestLogAnalyzer::Database
     line_type = line_type.name if line_type.respond_to?(:name)
     Object.const_get("#{line_type}_line".camelize)
   end
-  
+
   def default_classes
     [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning]
   end
-  
+
   # Loads the ORM classes by inspecting the tables in the current database
   def load_database_schema!
     connection.tables.map do |table|
@@ -39,16 +39,16 @@ class RequestLogAnalyzer::Database
       end
     end
   end
-  
+
   # Returns an array of all the ActiveRecord-bases ORM classes for this database
   def orm_classes
     default_classes + line_classes
   end
-  
+
   # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
   # a table name or a LineDefinition instance.
   def load_activerecord_class(linedefinition_or_table)
-    
+
     case linedefinition_or_table
     when String, Symbol
       klass_name = linedefinition_or_table.to_s.singularize.camelize
@@ -57,12 +57,12 @@ class RequestLogAnalyzer::Database
       klass_name = "#{linedefinition_or_table.name}_line".camelize
       klass      = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
     end
-    
+
     Object.const_set(klass_name, klass)
     klass = Object.const_get(klass_name)
     @line_classes << klass
     return klass
-  end  
+  end
 
   def fileformat_classes
     raise "No file_format provided!" unless file_format
@@ -70,25 +70,25 @@ class RequestLogAnalyzer::Database
     return default_classes + line_classes
   end
 
-  # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 
+  # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the
   # file format definition. These ORM classes will later be used to create records in the database.
   def create_database_schema!
     fileformat_classes.each { |klass| klass.create_table! }
   end
-  
+
   # Drops the table of all the ORM classes, and unregisters the classes
   def drop_database_schema!
     file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
     remove_orm_classes!
   end
-  
+
   # Registers the default ORM classes in the default namespace
   def register_default_orm_classes!
     Object.const_set('Request', RequestLogAnalyzer::Database::Request)
     Object.const_set('Source',  RequestLogAnalyzer::Database::Source)
     Object.const_set('Warning', RequestLogAnalyzer::Database::Warning)
   end
-  
+
   # Unregisters every ORM class constant
   def remove_orm_classes!
     orm_classes.each do |klass|

data/lib/request_log_analyzer/database/base.rb

@@ -1,5 +1,5 @@
 class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
-  
+
   self.abstract_class = true
 
   def <=>(other)
@@ -13,20 +13,20 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
   def line_type
     self.class.name.underscore.gsub(/_line$/, '').to_sym
   end
-  
+
   class_inheritable_accessor :line_definition
   cattr_accessor :database
 
   def self.subclass_from_line_definition(definition)
     klass = Class.new(RequestLogAnalyzer::Database::Base)
     klass.set_table_name("#{definition.name}_lines")
-    
+
     klass.line_definition = definition
-    
+
     # Set relations with requests and sources table
     klass.belongs_to :request
     klass.belongs_to :source
-    
+
     # Serialize complex fields into the database
     definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
       klass.send(:serialize, capture[:name], Hash)
@@ -48,32 +48,32 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
       klass.belongs_to :request
       RequestLogAnalyzer::Database::Request.has_many table.to_sym
     end
-    
+
     if klass.column_names.include?('source_id')
       klass.belongs_to :source
       RequestLogAnalyzer::Database::Source.has_many table.to_sym
     end
-    
+
     return klass
   end
-  
+
   def self.drop_table!
     database.connection.remove_index(self.table_name, [:source_id])  rescue nil
     database.connection.remove_index(self.table_name, [:request_id]) rescue nil
     database.connection.drop_table(self.table_name) if database.connection.table_exists?(self.table_name)
   end
-  
+
   def self.create_table!
     raise "No line_definition available to base table schema on!" unless self.line_definition
-    
+
     unless table_exists?
       database.connection.create_table(table_name.to_sym) do |t|
-      
+
         # Default fields
         t.column :request_id, :integer
         t.column :source_id,  :integer
         t.column :lineno,     :integer
-      
+
         line_definition.captures.each do |capture|
           # Add a field for every capture
           t.column(capture[:name], column_type(capture[:type]))
@@ -83,13 +83,13 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
         end
       end
     end
-    
+
     # Add indices to table for more speedy querying
     database.connection.add_index(self.table_name.to_sym, [:request_id]) # rescue
     database.connection.add_index(self.table_name.to_sym, [:source_id])  # rescue
   end
-  
-  
+
+
   # Function to determine the column type for a field
   # TODO: make more robust / include in file-format definition
   def self.column_type(type_indicator)
@@ -110,6 +110,6 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
     when :date;      :date
     else             :string
     end
-  end  
-  
+  end
+
 end

data/lib/request_log_analyzer/database/connection.rb

@@ -26,13 +26,13 @@ module RequestLogAnalyzer::Database::Connection
       raise "Cannot connect with this connection_identifier: #{connection_identifier.inspect}"
     end
   end
-  
+
   def disconnect
     RequestLogAnalyzer::Database::Base.remove_connection
   end
-  
+
   def connection
     RequestLogAnalyzer::Database::Base.connection
   end
-  
+
 end

data/lib/request_log_analyzer/database/request.rb

@@ -1,5 +1,5 @@
 class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
-  
+
   # Returns an array of all the Line objects of this request in the correct order.
   def lines
     @lines ||= begin
@@ -8,7 +8,7 @@ class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base
       lines.sort
     end
   end
-  
+
   # Creates the table to store requests in.
   def self.create_table!
     unless database.connection.table_exists?(:requests)

data/lib/request_log_analyzer/database/source.rb

@@ -1,5 +1,5 @@
 class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base
-  
+
   def self.create_table!
     unless database.connection.table_exists?(:sources)
       database.connection.create_table(:sources) do |t|

data/lib/request_log_analyzer/file_format.rb

@@ -1,9 +1,9 @@
 module RequestLogAnalyzer::FileFormat
-  
+
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
-  end  
-  
+  end
+
   # Loads a FileFormat::Base subclass instance.
   # You can provide:
   # * A FileFormat instance (which will return itself)
@@ -34,46 +34,46 @@ module RequestLogAnalyzer::FileFormat
 
     else
       # load a provided file format
-      klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))      
+      klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))
     end
-    
+
     # check the returned klass to see if it can be used
     raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
     raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
-    
+
     @current_file_format = klass.create(*args) # return an instance of the class
   end
 
-  # Base class for all log file format definitions. This class provides functions for subclasses to 
+  # Base class for all log file format definitions. This class provides functions for subclasses to
   # define their LineDefinitions and to define a summary report.
   #
   # A subclass of this class is instantiated when request-log-analyzer is started and this instance
   # is shared with all components of the application so they can act on the specifics of the format
   class Base
-    
+
     attr_reader :line_definitions, :report_trackers
-    
+
     ####################################################################################
     # CLASS METHODS for format definition
     ####################################################################################
-    
+
     # Registers the line definer instance for a subclass.
     def self.inherited(subclass)
       if subclass.superclass == RequestLogAnalyzer::FileFormat::Base
 
         # Create aline and report definer for this class
-        subclass.class_eval do 
+        subclass.class_eval do
           instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
           instance_variable_set(:@report_definer, RequestLogAnalyzer::Aggregator::Summarizer::Definer.new)
           class << self; attr_accessor :line_definer, :report_definer; end
-        end        
+        end
 
         # Create a custom Request class for this file format
         subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request)) unless subclass.const_defined?('Request')
       else
 
         # Copy the line and report definer from the parent class.
-        subclass.class_eval do 
+        subclass.class_eval do
           instance_variable_set(:@line_definer, superclass.line_definer.clone)
           instance_variable_set(:@report_definer, superclass.report_definer.clone)
           class << self; attr_accessor :line_definer, :report_definer; end
@@ -113,7 +113,7 @@ module RequestLogAnalyzer::FileFormat
       return self.new(line_definer.line_definitions, report_definer.trackers)
     end
 
-    def initialize(line_definitions = [], report_trackers = [])
+    def initialize(line_definitions = {}, report_trackers = [])
       @line_definitions, @report_trackers = line_definitions, report_trackers
     end
 
@@ -153,7 +153,7 @@ module RequestLogAnalyzer::FileFormat
         match = definition.matches(line, &warning_handler)
         return match if match
       end
-      
+
       return nil
     end
   end

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -1,20 +1,20 @@
 module RequestLogAnalyzer::FileFormat
 
-  # FileFormat for Amazon S3 access logs. 
+  # FileFormat for Amazon S3 access logs.
   #
   # Access logs are disabled by default on Amazon S3. To enable logging, see
   # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html
   class AmazonS3 < Base
-    
+
     line_definition :access do |line|
       line.header = true
       line.footer = true
       line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
-      line.captures << { :name => :bucket_owner,    :type => :string } << 
-                       { :name => :bucket,          :type => :string } << 
+      line.captures << { :name => :bucket_owner,    :type => :string } <<
+                       { :name => :bucket,          :type => :string } <<
                        { :name => :timestamp,       :type => :timestamp } <<
                        { :name => :remote_ip,       :type => :string } <<
-                       { :name => :requester,       :type => :string } << 
+                       { :name => :requester,       :type => :string } <<
                        { :name => :request_id,      :type => :string } <<
                        { :name => :operation,       :type => :string } <<
                        { :name => :key,             :type => :nillable_string } <<
@@ -28,44 +28,44 @@ module RequestLogAnalyzer::FileFormat
                        { :name => :referer,         :type => :referer } <<
                        { :name => :user_agent,      :type => :user_agent }
     end
-    
+
     report do |analyze|
       analyze.timespan
       analyze.hourly_spread
 
-      analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Most popular items"
-      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Request duration"
-      analyze.traffic  :traffic => :bytes_sent,  :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Traffic"
+      analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Most popular items"
+      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Request duration"
+      analyze.traffic  :traffic => :bytes_sent,  :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :title => "Traffic"
       analyze.frequency :category => :http_status, :title => 'HTTP status codes'
       analyze.frequency :category => :error_code, :title => 'Error codes'
     end
-    
+
     class Request < RequestLogAnalyzer::Request
-      
+
       MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
                 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
-      
+
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
         "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
-      
+
       # Make sure that the string '-' is parsed as a nil value.
       def convert_nillable_string(value, definition)
         value == '-' ? nil : value
       end
-      
+
       # Can be implemented in subclasses for improved categorizations
       def convert_referer(value, definition)
         value == '-' ? nil : value
       end
-      
+
       # Can be implemented in subclasses for improved categorizations
       def convert_user_agent(value, definition)
         value == '-' ? nil : value
       end
     end
-    
+
   end
 end

data/lib/request_log_analyzer/file_format/apache.rb

@@ -1,6 +1,6 @@
 module RequestLogAnalyzer::FileFormat
-  
-  # The Apache file format is able to log Apache access.log files. 
+
+  # The Apache file format is able to log Apache access.log files.
   #
   # The access.log can be configured in Apache to have many different formats. In theory, this
   # FileFormat can handle any format, but it must be aware of the log formatting that is used
@@ -22,7 +22,7 @@ module RequestLogAnalyzer::FileFormat
       :referer  => '%{Referer}i -> %U',
       :agent    => '%{User-agent}i'
     }
-    
+
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
       '%' => { :regexp => '%', :captures => [] },
@@ -48,8 +48,9 @@ module RequestLogAnalyzer::FileFormat
     # It will set up the line definition and the report trackers according to the Apache access log format,
     # which should be passed as first argument. By default, is uses the 'combined' log format.
     def self.create(*args)
-      access_line =  access_line_definition(args.first)
-      self.new({ :access => access_line}, report_trackers(access_line))
+      access_line = access_line_definition(args.first)
+      trackers = report_trackers(access_line) + report_definer.trackers
+      self.new(line_definer.line_definitions.merge(:access => access_line), trackers)
     end
 
     # Creates the access log line definition based on the Apache log format string
@@ -62,7 +63,7 @@ module RequestLogAnalyzer::FileFormat
       format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
 
         line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
-        
+
         if variable
           # Check if we recognize the log directive
           directive = LOG_DIRECTIVES[variable]
@@ -77,7 +78,7 @@ module RequestLogAnalyzer::FileFormat
           end
         end
       end
-      
+
       # Return a new line definition object
       return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
                                         :captures => captures, :header => true, :footer => true)
@@ -90,12 +91,12 @@ module RequestLogAnalyzer::FileFormat
       analyze.timespan      if line_definition.captures?(:timestamp)
       analyze.hourly_spread if line_definition.captures?(:timestamp)
 
-      analyze.frequency :category => :http_method, :amount => 20, :title => "HTTP methods"  if line_definition.captures?(:http_method)
-      analyze.frequency :category => :http_status, :amount => 20, :title => "HTTP statuses" if line_definition.captures?(:http_status)
-      analyze.frequency :category => lambda { |r| r.category }, :amount => 20, :title => "Most popular URIs"    if line_definition.captures?(:path)
+      analyze.frequency :category => :http_method, :title => "HTTP methods"  if line_definition.captures?(:http_method)
+      analyze.frequency :category => :http_status, :title => "HTTP statuses" if line_definition.captures?(:http_status)
+      analyze.frequency :category => lambda { |r| r.category }, :title => "Most popular URIs"    if line_definition.captures?(:path)
 
-      analyze.frequency :category => :user_agent, :amount => 20, :title => "User agents"    if line_definition.captures?(:user_agent)
-      analyze.frequency :category => :referer,    :amount => 20, :title => "Referers"       if line_definition.captures?(:referer)
+      analyze.frequency :category => :user_agent, :title => "User agents"    if line_definition.captures?(:user_agent)
+      analyze.frequency :category => :referer,    :title => "Referers"       if line_definition.captures?(:referer)
 
       analyze.duration :duration => :duration,  :category => lambda { |r| r.category }, :title => 'Request duration' if line_definition.captures?(:duration)
       analyze.traffic  :traffic => :bytes_sent, :category => lambda { |r| r.category }, :title => 'Traffic'          if line_definition.captures?(:bytes_sent)
@@ -105,32 +106,32 @@ module RequestLogAnalyzer::FileFormat
 
     # Define a custom Request class for the Apache file format to speed up timestamp handling.
     class Request < RequestLogAnalyzer::Request
-      
+
       def category
         first(:path)
       end
-      
+
       MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
                 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
-      
+
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
         "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
       end
-      
+
       # This function can be overridden to rewrite the path for better categorization in the
       # reports.
       def convert_path(value, definition)
         value
       end
-      
-      # This function can be overridden to simplify the user agent string for better 
+
+      # This function can be overridden to simplify the user agent string for better
       # categorization in the reports
       def convert_user_agent(value, definition)
         value # TODO
       end
-      
+
       # Make sure that the string '-' is parsed as a nil value.
       def convert_nillable_string(value, definition)
         value == '-' ? nil : value