request-log-analyzer 1.1.1 → 1.1.2

data/bin/request-log-analyzer

@@ -27,14 +27,6 @@ begin
       strip.switch(:keep_junk_lines, :j)      
     end
     
-    command_line.command(:anonymize) do |anonymize|
-      anonymize.minimum_parameters = 1
-      anonymize.option(:format, :alias => :f, :default => 'rails')            
-      anonymize.option(:output, :alias => :o)
-      anonymize.switch(:discard_teaser_lines, :t)
-      anonymize.switch(:keep_junk_lines, :j)      
-    end
-    
     command_line.option(:format, :alias => :f, :default => 'rails')
     command_line.option(:file, :alias => :e)
     command_line.switch(:assume_correct_order)
@@ -111,9 +103,6 @@ when :install
 when :strip
   require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'  
   RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
-when :anonymize
-  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'    
-  RequestLogAnalyzer::LogProcessor.build(:anonymize, arguments).run!  
 else 
   puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"  
   puts "Website: http://github.com/wvanbergen/request-log-analyzer"

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -12,15 +12,19 @@ module RequestLogAnalyzer::Aggregator
         @trackers = []
       end
       
+      def reset!
+        @trackers = []
+      end
+      
       def method_missing(tracker_method, *args)
         track(tracker_method, args.first)
       end
       
-      def category(category_field, options = {})
+      def frequency(category_field, options = {})
         if category_field.kind_of?(Symbol)
-          track(:category, options.merge(:category => category_field))
+          track(:frequency, options.merge(:category => category_field))
         elsif category_field.kind_of?(Hash)
-          track(:category, category_field.merge(options))
+          track(:frequency, category_field.merge(options))
         end
       end
       

data/lib/request_log_analyzer/controller.rb

@@ -131,7 +131,7 @@ module RequestLogAnalyzer
     def handle_progress(message, value = nil)
       case message
       when :started
-        @progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value))
+        @progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value), STDOUT)
       when :finished
         @progress_bar.finish
         @progress_bar = nil
@@ -160,6 +160,18 @@ module RequestLogAnalyzer
       @filters << filter.new(file_format, @options.merge(filter_options))
     end
     
+    def filter_request(request)
+      @filters.each do |filter| 
+        request = filter.filter(request)
+        return nil if request.nil?
+      end
+      return request
+    end
+    
+    def aggregate_request(request)
+      @aggregators.each { |agg| agg.aggregate(request) }
+    end
+    
     # Runs RequestLogAnalyzer
     # 1. Call prepare on every aggregator
     # 2. Generate requests from source object
@@ -175,8 +187,8 @@ module RequestLogAnalyzer
       
       begin
         @source.each_request do |request|
-          @filters.each { |filter| request = filter.filter(request) }
-          @aggregators.each { |agg| agg.aggregate(request) } if request
+          request = filter_request(request)
+          aggregate_request(request) unless request.nil?
         end
       rescue Interrupt => e
         handle_progress(:interrupted)

data/lib/request_log_analyzer/file_format.rb

@@ -1,6 +1,5 @@
 module RequestLogAnalyzer::FileFormat
   
-  
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end  
@@ -24,14 +23,24 @@ module RequestLogAnalyzer::FileFormat
     elsif file_format.kind_of?(String) && File.exist?(file_format)
       # load a format from a ruby file
       require file_format
-      klass = Object.const_get(RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb')))
+      const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
+      if RequestLogAnalyzer::FileFormat.const_defined?(const)
+        klass = RequestLogAnalyzer::FileFormat.const_get(const)
+      elsif Object.const_defined?(const)
+        klass = Object.const_get(const)
+      else
+        raise "Cannot load class #{const} from #{file_format}!"
+      end
 
     else
       # load a provided file format
       klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))      
     end
     
-    raise if klass.nil?
+    # check the returned klass to see if it can be used
+    raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
+    raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
+    
     @current_file_format = klass.new # return an instance of the class
   end  
   
@@ -56,13 +65,29 @@ module RequestLogAnalyzer::FileFormat
       
     # Registers the line definer instance for a subclass.
     def self.inherited(subclass)
-       subclass.instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
-       subclass.class_eval { class << self; attr_accessor :line_definer; end } 
-       subclass.class_eval { class << self; attr_accessor :report_definer; end }
-       
-       unless subclass.const_defined?('Request')
-         subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request))         
-       end
+      if subclass.superclass == RequestLogAnalyzer::FileFormat::Base
+        
+        # Create aline and report definer for this class
+        subclass.class_eval do 
+          instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
+          instance_variable_set(:@report_definer, RequestLogAnalyzer::Aggregator::Summarizer::Definer.new)
+          class << self; attr_accessor :line_definer, :report_definer; end
+        end        
+
+        # Create a custom Request class for this file format
+        subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request)) unless subclass.const_defined?('Request')
+      else
+        
+        # Copy the line and report definer from the parent class.
+        subclass.class_eval do 
+          instance_variable_set(:@line_definer, superclass.line_definer)
+          instance_variable_set(:@report_definer, superclass.report_definer)
+          class << self; attr_accessor :line_definer, :report_definer; end
+        end        
+        
+        # Create a custom Request class based on the superclass's Request class
+        subclass.const_set('Request', Class.new(subclass.superclass::Request)) unless subclass.const_defined?('Request')
+      end
     end    
     
     # Specifies a single line defintions.
@@ -77,16 +102,19 @@ module RequestLogAnalyzer::FileFormat
     # Specifies multiple line definitions at once using a block
     def self.format_definition(&block)
       if block_given?
-        yield(@line_definer) 
+        yield self.line_definer
       else
-        return @line_definer
+        return self.line_definer
       end
     end
     
     # Specifies the summary report using a block.
-    def self.report(&block)
-      @report_definer = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
-      yield(@report_definer)
+    def self.report(mode = :append, &block)
+      if mode == :overwrite
+        self.report_definer.reset!
+      end
+      
+      yield(self.report_definer)
     end
 
     # Returns all line definitions
@@ -96,7 +124,7 @@ module RequestLogAnalyzer::FileFormat
     
     # Returns all the defined trackers for the summary report.
     def report_trackers
-      self.class.instance_variable_get(:@report_definer).trackers rescue []
+      self.class.report_definer.trackers# =>  rescue []
     end
     
     # Checks whether the line definitions form a valid language.

data/lib/request_log_analyzer/file_format/merb.rb

@@ -7,7 +7,7 @@ module RequestLogAnalyzer::FileFormat
       line.header = true
       line.teaser = /Started/
       line.regexp = /Started request handling\:\ (.+)/
-      line.captures << { :name => :timestamp, :type => :timestamp, :anonymize => :slightly }
+      line.captures << { :name => :timestamp, :type => :timestamp }
     end    
     
     # ~ Params: {"action"=>"create", "controller"=>"session"}
@@ -23,10 +23,10 @@ module RequestLogAnalyzer::FileFormat
       line.footer = true
       line.teaser = /\{:dispatch_time/
       line.regexp = /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/
-      line.captures << { :name => :dispatch_time,       :type => :sec, :anonymize => :slightly } \
-                    << { :name => :after_filters_time,  :type => :sec, :anonymize => :slightly } \
-                    << { :name => :before_filters_time, :type => :sec, :anonymize => :slightly } \
-                    << { :name => :action_time,         :type => :sec, :anonymize => :slightly }
+      line.captures << { :name => :dispatch_time,       :type => :duration } \
+                    << { :name => :after_filters_time,  :type => :duration } \
+                    << { :name => :before_filters_time, :type => :duration } \
+                    << { :name => :action_time,         :type => :duration }
     end
     
     

data/lib/request_log_analyzer/file_format/rails.rb

@@ -9,9 +9,9 @@ module RequestLogAnalyzer::FileFormat
       line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
       line.captures << { :name => :controller, :type  => :string } \
                     << { :name => :action,     :type  => :string } \
-                    << { :name => :format,     :type  => :string } \
-                    << { :name => :ip,         :type  => :string, :anonymize => :ip } \
-                    << { :name => :timestamp,  :type  => :timestamp, :anonymize => :slightly } \
+                    << { :name => :format,     :type  => :format } \
+                    << { :name => :ip,         :type  => :string } \
+                    << { :name => :timestamp,  :type  => :timestamp } \
                     << { :name => :method,     :type  => :string }
     end
 
@@ -28,7 +28,7 @@ module RequestLogAnalyzer::FileFormat
                     << { :name => :message,     :type => :string } \
                     << { :name => :line,        :type => :integer } \
                     << { :name => :file,        :type => :string } \
-                    << { :name => :stack_trace, :type => :string, :anonymize => true }
+                    << { :name => :stack_trace, :type => :string }
     end
 
 
@@ -49,42 +49,56 @@ module RequestLogAnalyzer::FileFormat
       line.teaser = /Completed in /
       line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
     
-      line.captures << { :name => :duration, :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :view,     :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :db,       :type => :sec,    :anonymize => :slightly } \
+      line.captures << { :name => :duration, :type => :duration, :unit => :sec } \
+                    << { :name => :view,     :type => :duration, :unit => :sec } \
+                    << { :name => :db,       :type => :duration, :unit => :sec } \
                     << { :name => :status,   :type => :integer } \
-                    << { :name => :url,      :type => :string, :anonymize => :url } # Old variant 
+                    << { :name => :url,      :type => :string } # Old variant 
                   
-      line.captures << { :name => :duration, :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :view,     :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :db,       :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :status,   :type => :integer} \
-                    << { :name => :url,      :type => :string, :anonymize => :url }  # 2.2 variant 
+      line.captures << { :name => :duration, :type => :duration, :unit => :msec } \
+                    << { :name => :view,     :type => :duration, :unit => :msec } \
+                    << { :name => :db,       :type => :duration, :unit => :msec } \
+                    << { :name => :status,   :type => :integer } \
+                    << { :name => :url,      :type => :string }  # 2.2 variant 
     end
 
 
 
     REQUEST_CATEGORIZER = Proc.new do |request|
-      format = request[:format] || 'html'
-      "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+      "#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
     end
 
     report do |analyze|
       analyze.timespan :line_type => :processing
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
-      analyze.category :method, :title => 'HTTP methods'
-      analyze.category :status, :title => 'HTTP statuses returned'
-      analyze.category :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
     
       analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
       analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
       analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
     
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
               :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
             
       analyze.hourly_spread :line_type => :processing
-      analyze.category :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+      analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+    end
+
+    # Define a custom Request class for the Rails file format to speed up timestamp handling
+    # and to ensure that a format is always set.
+    class Request < RequestLogAnalyzer::Request
+
+      # Do not use DateTime.parse
+      def convert_timestamp(value, definition)
+        value.gsub(/[^0-9]/, '')[0...14].to_i unless value.nil?
+      end
+      
+      # Set 'html' as default format for a request
+      def convert_format(value, definition)
+        value || 'html'
+      end
     end
 
   end

data/lib/request_log_analyzer/file_format/rails_development.rb

@@ -10,8 +10,8 @@ module RequestLogAnalyzer::FileFormat
       line.captures << { :name => :controller, :type  => :string } \
                     << { :name => :action,     :type  => :string } \
                     << { :name => :format,     :type  => :string } \
-                    << { :name => :ip,         :type  => :string, :anonymize => :ip } \
-                    << { :name => :timestamp,  :type  => :timestamp, :anonymize => :slightly } \
+                    << { :name => :ip,         :type  => :string } \
+                    << { :name => :timestamp,  :type  => :timestamp } \
                     << { :name => :method,     :type  => :string }
     end
 
@@ -25,14 +25,14 @@ module RequestLogAnalyzer::FileFormat
       line.teaser = /Rendered /
       line.regexp = /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/
       line.captures << { :name => :render_file,     :type  => :string } \
-                    << { :name => :render_duration, :type  => :msec }
+                    << { :name => :render_duration, :type  => :duration, :unit => :msec }
     end
   
     #  User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) 
     line_definition :query_executed do |line|
       line.regexp = /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?(.+) (?:\e\[0m)?/
       line.captures << { :name => :query_class,    :type  => :string } \
-                    << { :name => :query_duration, :type  => :msec } \
+                    << { :name => :query_duration, :type  => :duration, :unit => :msec } \
                     << { :name => :query_sql,      :type  => :string }
     end
 
@@ -40,7 +40,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :query_cached do |line|
       line.teaser = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?(.+) (?:\e\[0m)?/  
       line.regexp = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?(.+) (?:\e\[0m)?/
-      line.captures << { :name => :cached_duration, :type  => :msec } \
+      line.captures << { :name => :cached_duration, :type  => :duration, :unit => :msec } \
                     << { :name => :cached_sql,      :type  => :string }
     end  
   
@@ -52,7 +52,7 @@ module RequestLogAnalyzer::FileFormat
                     << { :name => :message,     :type => :string } \
                     << { :name => :line,        :type => :integer } \
                     << { :name => :file,        :type => :string } \
-                    << { :name => :stack_trace, :type => :string, :anonymize => true }
+                    << { :name => :stack_trace, :type => :string }
     end
 
 
@@ -73,17 +73,17 @@ module RequestLogAnalyzer::FileFormat
       line.teaser = /Completed in /
       line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
     
-      line.captures << { :name => :duration, :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :view,     :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :db,       :type => :sec,    :anonymize => :slightly } \
+      line.captures << { :name => :duration, :type => :duration } \
+                    << { :name => :view,     :type => :duration } \
+                    << { :name => :db,       :type => :duration } \
                     << { :name => :status,   :type => :integer } \
-                    << { :name => :url,      :type => :string, :anonymize => :url } # Old variant 
+                    << { :name => :url,      :type => :string } # Old variant 
                   
-      line.captures << { :name => :duration, :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :view,     :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :db,       :type => :msec,   :anonymize => :slightly } \
+      line.captures << { :name => :duration, :type => :duration, :unit => :msec } \
+                    << { :name => :view,     :type => :duration, :unit => :msec } \
+                    << { :name => :db,       :type => :duration, :unit => :msec } \
                     << { :name => :status,   :type => :integer} \
-                    << { :name => :url,      :type => :string, :anonymize => :url }  # 2.2 variant 
+                    << { :name => :url,      :type => :string }  # 2.2 variant 
     end
 
     REQUEST_CATEGORIZER = Proc.new do |request|
@@ -93,20 +93,27 @@ module RequestLogAnalyzer::FileFormat
 
     report do |analyze|
       analyze.timespan :line_type => :processing
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
-      analyze.category :method, :title => 'HTTP methods'
-      analyze.category :status, :title => 'HTTP statuses returned'
-      analyze.category :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
     
       analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
       analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
       analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
     
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
               :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
             
       analyze.hourly_spread :line_type => :processing
-      analyze.category :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+      analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
     end
+    
+    class Request < RequestLogAnalyzer::Request
+
+      def convert_timestamp(value, definition)
+        value.gsub(/[^0-9]/)[0...14].to_i unless value.nil?
+      end
+    end    
   end
 end

data/lib/request_log_analyzer/filter/{anonimize.rb → anonymize.rb}

@@ -1,11 +1,11 @@
 module RequestLogAnalyzer::Filter
   
-  # Filter to select or reject a specific field
+  # Filter to anonymize parsed values
   # Options
   # * <tt>:mode</tt> :reject or :accept.
   # * <tt>:field</tt> Specific field to accept or reject.
   # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
-  class Anonimize < Base
+  class Anonymize < Base
    
     def prepare
     end
@@ -22,9 +22,7 @@ module RequestLogAnalyzer::Filter
       value * ((75 + rand(50)) / 100.0)
     end
 
-    def filter(request)
-      return nil unless request
-      
+    def filter(request)    
       request.attributes.each do |key, value|
         if key == :ip
           request.attributes[key] = generate_random_ip

data/lib/request_log_analyzer/filter/field.rb

@@ -27,13 +27,9 @@ module RequestLogAnalyzer::Filter
     # Returns nil otherwise.
     # <tt>request</tt> Request Object
     def filter(request)
-      return nil unless request
-      
-      found_field = request.every(@field).any? { |value| @value === value }
-      
+      found_field = request.every(@field).any? { |value| @value === value.to_s }      
       return nil if !found_field && @mode == :select
       return nil if found_field && @mode == :reject
-      
       return request
     end 
   end