wvanbergen-request-log-analyzer 1.1.1 → 1.1.2

data/bin/request-log-analyzer

@@ -27,14 +27,6 @@ begin
       strip.switch(:keep_junk_lines, :j)      
     end
     
-    command_line.command(:anonymize) do |anonymize|
-      anonymize.minimum_parameters = 1
-      anonymize.option(:format, :alias => :f, :default => 'rails')            
-      anonymize.option(:output, :alias => :o)
-      anonymize.switch(:discard_teaser_lines, :t)
-      anonymize.switch(:keep_junk_lines, :j)      
-    end
-    
     command_line.option(:format, :alias => :f, :default => 'rails')
     command_line.option(:file, :alias => :e)
     command_line.switch(:assume_correct_order)
@@ -111,9 +103,6 @@ when :install
 when :strip
   require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'  
   RequestLogAnalyzer::LogProcessor.build(:strip, arguments).run!
-when :anonymize
-  require File.dirname(__FILE__) + '/../lib/request_log_analyzer/log_processor'    
-  RequestLogAnalyzer::LogProcessor.build(:anonymize, arguments).run!  
 else 
   puts "Request-log-analyzer, by Willem van Bergen and Bart ten Brinke - version #{RequestLogAnalyzer::VERSION}"  
   puts "Website: http://github.com/wvanbergen/request-log-analyzer"

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -12,15 +12,19 @@ module RequestLogAnalyzer::Aggregator
         @trackers = []
       end
       
+      def reset!
+        @trackers = []
+      end
+      
       def method_missing(tracker_method, *args)
         track(tracker_method, args.first)
       end
       
-      def category(category_field, options = {})
+      def frequency(category_field, options = {})
         if category_field.kind_of?(Symbol)
-          track(:category, options.merge(:category => category_field))
+          track(:frequency, options.merge(:category => category_field))
         elsif category_field.kind_of?(Hash)
-          track(:category, category_field.merge(options))
+          track(:frequency, category_field.merge(options))
         end
       end
       

data/lib/request_log_analyzer/controller.rb

@@ -131,7 +131,7 @@ module RequestLogAnalyzer
     def handle_progress(message, value = nil)
       case message
       when :started
-        @progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value))
+        @progress_bar = CommandLine::ProgressBar.new(File.basename(value), File.size(value), STDOUT)
       when :finished
         @progress_bar.finish
         @progress_bar = nil
@@ -160,6 +160,18 @@ module RequestLogAnalyzer
       @filters << filter.new(file_format, @options.merge(filter_options))
     end
     
+    def filter_request(request)
+      @filters.each do |filter| 
+        request = filter.filter(request)
+        return nil if request.nil?
+      end
+      return request
+    end
+    
+    def aggregate_request(request)
+      @aggregators.each { |agg| agg.aggregate(request) }
+    end
+    
     # Runs RequestLogAnalyzer
     # 1. Call prepare on every aggregator
     # 2. Generate requests from source object
@@ -175,8 +187,8 @@ module RequestLogAnalyzer
       
       begin
         @source.each_request do |request|
-          @filters.each { |filter| request = filter.filter(request) }
-          @aggregators.each { |agg| agg.aggregate(request) } if request
+          request = filter_request(request)
+          aggregate_request(request) unless request.nil?
         end
       rescue Interrupt => e
         handle_progress(:interrupted)

data/lib/request_log_analyzer/file_format/merb.rb

@@ -7,7 +7,7 @@ module RequestLogAnalyzer::FileFormat
       line.header = true
       line.teaser = /Started/
       line.regexp = /Started request handling\:\ (.+)/
-      line.captures << { :name => :timestamp, :type => :timestamp, :anonymize => :slightly }
+      line.captures << { :name => :timestamp, :type => :timestamp }
     end    
     
     # ~ Params: {"action"=>"create", "controller"=>"session"}
@@ -23,10 +23,10 @@ module RequestLogAnalyzer::FileFormat
       line.footer = true
       line.teaser = /\{:dispatch_time/
       line.regexp = /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/
-      line.captures << { :name => :dispatch_time,       :type => :sec, :anonymize => :slightly } \
-                    << { :name => :after_filters_time,  :type => :sec, :anonymize => :slightly } \
-                    << { :name => :before_filters_time, :type => :sec, :anonymize => :slightly } \
-                    << { :name => :action_time,         :type => :sec, :anonymize => :slightly }
+      line.captures << { :name => :dispatch_time,       :type => :duration } \
+                    << { :name => :after_filters_time,  :type => :duration } \
+                    << { :name => :before_filters_time, :type => :duration } \
+                    << { :name => :action_time,         :type => :duration }
     end
     
     

data/lib/request_log_analyzer/file_format/rails.rb

@@ -9,9 +9,9 @@ module RequestLogAnalyzer::FileFormat
       line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
       line.captures << { :name => :controller, :type  => :string } \
                     << { :name => :action,     :type  => :string } \
-                    << { :name => :format,     :type  => :string } \
-                    << { :name => :ip,         :type  => :string, :anonymize => :ip } \
-                    << { :name => :timestamp,  :type  => :timestamp, :anonymize => :slightly } \
+                    << { :name => :format,     :type  => :format } \
+                    << { :name => :ip,         :type  => :string } \
+                    << { :name => :timestamp,  :type  => :timestamp } \
                     << { :name => :method,     :type  => :string }
     end
 
@@ -28,7 +28,7 @@ module RequestLogAnalyzer::FileFormat
                     << { :name => :message,     :type => :string } \
                     << { :name => :line,        :type => :integer } \
                     << { :name => :file,        :type => :string } \
-                    << { :name => :stack_trace, :type => :string, :anonymize => true }
+                    << { :name => :stack_trace, :type => :string }
     end
 
 
@@ -49,42 +49,56 @@ module RequestLogAnalyzer::FileFormat
       line.teaser = /Completed in /
       line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
     
-      line.captures << { :name => :duration, :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :view,     :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :db,       :type => :sec,    :anonymize => :slightly } \
+      line.captures << { :name => :duration, :type => :duration, :unit => :sec } \
+                    << { :name => :view,     :type => :duration, :unit => :sec } \
+                    << { :name => :db,       :type => :duration, :unit => :sec } \
                     << { :name => :status,   :type => :integer } \
-                    << { :name => :url,      :type => :string, :anonymize => :url } # Old variant 
+                    << { :name => :url,      :type => :string } # Old variant 
                   
-      line.captures << { :name => :duration, :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :view,     :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :db,       :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :status,   :type => :integer} \
-                    << { :name => :url,      :type => :string, :anonymize => :url }  # 2.2 variant 
+      line.captures << { :name => :duration, :type => :duration, :unit => :msec } \
+                    << { :name => :view,     :type => :duration, :unit => :msec } \
+                    << { :name => :db,       :type => :duration, :unit => :msec } \
+                    << { :name => :status,   :type => :integer } \
+                    << { :name => :url,      :type => :string }  # 2.2 variant 
     end
 
 
 
     REQUEST_CATEGORIZER = Proc.new do |request|
-      format = request[:format] || 'html'
-      "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+      "#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
     end
 
     report do |analyze|
       analyze.timespan :line_type => :processing
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
-      analyze.category :method, :title => 'HTTP methods'
-      analyze.category :status, :title => 'HTTP statuses returned'
-      analyze.category :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
     
       analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
       analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
       analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
     
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
               :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
             
       analyze.hourly_spread :line_type => :processing
-      analyze.category :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+      analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+    end
+
+    # Define a custom Request class for the Rails file format to speed up timestamp handling
+    # and to ensure that a format is always set.
+    class Request < RequestLogAnalyzer::Request
+
+      # Do not use DateTime.parse
+      def convert_timestamp(value, definition)
+        value.gsub(/[^0-9]/, '')[0...14].to_i unless value.nil?
+      end
+      
+      # Set 'html' as default format for a request
+      def convert_format(value, definition)
+        value || 'html'
+      end
     end
 
   end

data/lib/request_log_analyzer/file_format/rails_development.rb

@@ -10,8 +10,8 @@ module RequestLogAnalyzer::FileFormat
       line.captures << { :name => :controller, :type  => :string } \
                     << { :name => :action,     :type  => :string } \
                     << { :name => :format,     :type  => :string } \
-                    << { :name => :ip,         :type  => :string, :anonymize => :ip } \
-                    << { :name => :timestamp,  :type  => :timestamp, :anonymize => :slightly } \
+                    << { :name => :ip,         :type  => :string } \
+                    << { :name => :timestamp,  :type  => :timestamp } \
                     << { :name => :method,     :type  => :string }
     end
 
@@ -25,14 +25,14 @@ module RequestLogAnalyzer::FileFormat
       line.teaser = /Rendered /
       line.regexp = /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/
       line.captures << { :name => :render_file,     :type  => :string } \
-                    << { :name => :render_duration, :type  => :msec }
+                    << { :name => :render_duration, :type  => :duration, :unit => :msec }
     end
   
     #  User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) 
     line_definition :query_executed do |line|
       line.regexp = /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?(.+) (?:\e\[0m)?/
       line.captures << { :name => :query_class,    :type  => :string } \
-                    << { :name => :query_duration, :type  => :msec } \
+                    << { :name => :query_duration, :type  => :duration, :unit => :msec } \
                     << { :name => :query_sql,      :type  => :string }
     end
 
@@ -40,7 +40,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :query_cached do |line|
       line.teaser = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?(.+) (?:\e\[0m)?/  
       line.regexp = /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?(.+) (?:\e\[0m)?/
-      line.captures << { :name => :cached_duration, :type  => :msec } \
+      line.captures << { :name => :cached_duration, :type  => :duration, :unit => :msec } \
                     << { :name => :cached_sql,      :type  => :string }
     end  
   
@@ -52,7 +52,7 @@ module RequestLogAnalyzer::FileFormat
                     << { :name => :message,     :type => :string } \
                     << { :name => :line,        :type => :integer } \
                     << { :name => :file,        :type => :string } \
-                    << { :name => :stack_trace, :type => :string, :anonymize => true }
+                    << { :name => :stack_trace, :type => :string }
     end
 
 
@@ -73,17 +73,17 @@ module RequestLogAnalyzer::FileFormat
       line.teaser = /Completed in /
       line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
     
-      line.captures << { :name => :duration, :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :view,     :type => :sec,    :anonymize => :slightly } \
-                    << { :name => :db,       :type => :sec,    :anonymize => :slightly } \
+      line.captures << { :name => :duration, :type => :duration } \
+                    << { :name => :view,     :type => :duration } \
+                    << { :name => :db,       :type => :duration } \
                     << { :name => :status,   :type => :integer } \
-                    << { :name => :url,      :type => :string, :anonymize => :url } # Old variant 
+                    << { :name => :url,      :type => :string } # Old variant 
                   
-      line.captures << { :name => :duration, :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :view,     :type => :msec,   :anonymize => :slightly } \
-                    << { :name => :db,       :type => :msec,   :anonymize => :slightly } \
+      line.captures << { :name => :duration, :type => :duration, :unit => :msec } \
+                    << { :name => :view,     :type => :duration, :unit => :msec } \
+                    << { :name => :db,       :type => :duration, :unit => :msec } \
                     << { :name => :status,   :type => :integer} \
-                    << { :name => :url,      :type => :string, :anonymize => :url }  # 2.2 variant 
+                    << { :name => :url,      :type => :string }  # 2.2 variant 
     end
 
     REQUEST_CATEGORIZER = Proc.new do |request|
@@ -93,20 +93,27 @@ module RequestLogAnalyzer::FileFormat
 
     report do |analyze|
       analyze.timespan :line_type => :processing
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
-      analyze.category :method, :title => 'HTTP methods'
-      analyze.category :status, :title => 'HTTP statuses returned'
-      analyze.category :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      analyze.frequency :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
     
       analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
       analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
       analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
     
-      analyze.category :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
               :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
             
       analyze.hourly_spread :line_type => :processing
-      analyze.category :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+      analyze.frequency :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
     end
+    
+    class Request < RequestLogAnalyzer::Request
+
+      def convert_timestamp(value, definition)
+        value.gsub(/[^0-9]/)[0...14].to_i unless value.nil?
+      end
+    end    
   end
 end

data/lib/request_log_analyzer/file_format.rb

@@ -1,6 +1,5 @@
 module RequestLogAnalyzer::FileFormat
   
-  
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end  
@@ -24,14 +23,24 @@ module RequestLogAnalyzer::FileFormat
     elsif file_format.kind_of?(String) && File.exist?(file_format)
       # load a format from a ruby file
       require file_format
-      klass = Object.const_get(RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb')))
+      const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
+      if RequestLogAnalyzer::FileFormat.const_defined?(const)
+        klass = RequestLogAnalyzer::FileFormat.const_get(const)
+      elsif Object.const_defined?(const)
+        klass = Object.const_get(const)
+      else
+        raise "Cannot load class #{const} from #{file_format}!"
+      end
 
     else
       # load a provided file format
       klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))      
     end
     
-    raise if klass.nil?
+    # check the returned klass to see if it can be used
+    raise "Could not load a file format from #{file_format.inspect}" if klass.nil?
+    raise "Invalid FileFormat class" unless klass.kind_of?(Class) && klass.ancestors.include?(RequestLogAnalyzer::FileFormat::Base)
+    
     @current_file_format = klass.new # return an instance of the class
   end  
   
@@ -56,13 +65,29 @@ module RequestLogAnalyzer::FileFormat
       
     # Registers the line definer instance for a subclass.
     def self.inherited(subclass)
-       subclass.instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
-       subclass.class_eval { class << self; attr_accessor :line_definer; end } 
-       subclass.class_eval { class << self; attr_accessor :report_definer; end }
-       
-       unless subclass.const_defined?('Request')
-         subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request))         
-       end
+      if subclass.superclass == RequestLogAnalyzer::FileFormat::Base
+        
+        # Create aline and report definer for this class
+        subclass.class_eval do 
+          instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
+          instance_variable_set(:@report_definer, RequestLogAnalyzer::Aggregator::Summarizer::Definer.new)
+          class << self; attr_accessor :line_definer, :report_definer; end
+        end        
+
+        # Create a custom Request class for this file format
+        subclass.const_set('Request', Class.new(RequestLogAnalyzer::Request)) unless subclass.const_defined?('Request')
+      else
+        
+        # Copy the line and report definer from the parent class.
+        subclass.class_eval do 
+          instance_variable_set(:@line_definer, superclass.line_definer)
+          instance_variable_set(:@report_definer, superclass.report_definer)
+          class << self; attr_accessor :line_definer, :report_definer; end
+        end        
+        
+        # Create a custom Request class based on the superclass's Request class
+        subclass.const_set('Request', Class.new(subclass.superclass::Request)) unless subclass.const_defined?('Request')
+      end
     end    
     
     # Specifies a single line defintions.
@@ -77,16 +102,19 @@ module RequestLogAnalyzer::FileFormat
     # Specifies multiple line definitions at once using a block
     def self.format_definition(&block)
       if block_given?
-        yield(@line_definer) 
+        yield self.line_definer
       else
-        return @line_definer
+        return self.line_definer
       end
     end
     
     # Specifies the summary report using a block.
-    def self.report(&block)
-      @report_definer = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
-      yield(@report_definer)
+    def self.report(mode = :append, &block)
+      if mode == :overwrite
+        self.report_definer.reset!
+      end
+      
+      yield(self.report_definer)
     end
 
     # Returns all line definitions
@@ -96,7 +124,7 @@ module RequestLogAnalyzer::FileFormat
     
     # Returns all the defined trackers for the summary report.
     def report_trackers
-      self.class.instance_variable_get(:@report_definer).trackers rescue []
+      self.class.report_definer.trackers# =>  rescue []
     end
     
     # Checks whether the line definitions form a valid language.

data/lib/request_log_analyzer/filter/{anonimize.rb → anonymize.rb}

@@ -1,11 +1,11 @@
 module RequestLogAnalyzer::Filter
   
-  # Filter to select or reject a specific field
+  # Filter to anonymize parsed values
   # Options
   # * <tt>:mode</tt> :reject or :accept.
   # * <tt>:field</tt> Specific field to accept or reject.
   # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
-  class Anonimize < Base
+  class Anonymize < Base
    
     def prepare
     end
@@ -22,9 +22,7 @@ module RequestLogAnalyzer::Filter
       value * ((75 + rand(50)) / 100.0)
     end
 
-    def filter(request)
-      return nil unless request
-      
+    def filter(request)    
       request.attributes.each do |key, value|
         if key == :ip
           request.attributes[key] = generate_random_ip

data/lib/request_log_analyzer/filter/field.rb

@@ -27,13 +27,9 @@ module RequestLogAnalyzer::Filter
     # Returns nil otherwise.
     # <tt>request</tt> Request Object
     def filter(request)
-      return nil unless request
-      
-      found_field = request.every(@field).any? { |value| @value === value }
-      
+      found_field = request.every(@field).any? { |value| @value === value.to_s }      
       return nil if !found_field && @mode == :select
       return nil if found_field && @mode == :reject
-      
       return request
     end 
   end

data/lib/request_log_analyzer/filter/timespan.rb

@@ -22,8 +22,6 @@ module RequestLogAnalyzer::Filter
     # Returns nil otherwise
     # <tt>request</tt> Request object.
     def filter(request)
-      return nil unless request
-      
       if @after && @before && request.timestamp <= @before && @after <= request.timestamp
         return request
       elsif @after && @before.nil? && @after <= request.timestamp

data/lib/request_log_analyzer/line_definition.rb

@@ -1,22 +1,10 @@
 module RequestLogAnalyzer
   
-  module Anonymizers
-    def anonymizer_for_ip(value, capture_definition)
-      '127.0.0.1'
-    end
-
-    def anonymizer_for_url(value, capture_definition)
-      value.sub(/^https?\:\/\/[A-Za-z0-9\.-]+\//, "http://example.com/")
-    end
-  end  
-  
   # The line definition class is used to specify what lines should be parsed from the log file.
   # It contains functionality to match a line against the definition and parse the information
   # from this line. This is used by the LogParser class when parsing a log file..
   class LineDefinition
 
-    include RequestLogAnalyzer::Anonymizers
-
     class Definer
       
       attr_accessor :line_definitions
@@ -52,20 +40,6 @@ module RequestLogAnalyzer
       return definition
     end
     
-    # Converts a parsed value (String) to the desired value using some heuristics.
-    def convert_value(value, type)
-      case type
-      when :integer;   value.to_i
-      when :float;     value.to_f
-      when :decimal;   value.to_f          
-      when :symbol;    value.to_sym
-      when :sec;       value.to_f
-      when :msec;      value.to_f / 1000
-      when :timestamp; value.gsub(/[^0-9]/,'')[0..13].to_i # Retrieve with: DateTime.parse(value, '%Y%m%d%H%M%S')
-      else value
-      end
-    end
-    
     # Checks whether a given line matches this definition. 
     # It will return false if a line does not match. If the line matches, a hash is returned
     # with all the fields parsed from that line as content.
@@ -74,17 +48,7 @@ module RequestLogAnalyzer
     def matches(line, lineno = nil, parser = nil)
       if @teaser.nil? || @teaser =~ line
         if match_data = line.match(@regexp)
-          request_info = { :line_type => name, :lineno => lineno }
-          
-          captures.each_with_index do |capture, index|
-            next if capture == :ignore
-
-            if match_data.captures[index]
-              request_info[capture[:name]] = convert_value(match_data.captures[index], capture[:type])
-            end
-
-          end
-          return request_info
+          return { :line_definition => self, :lineno => lineno, :captures => match_data.captures}
         else
           if @teaser && parser
             parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")
@@ -97,63 +61,23 @@ module RequestLogAnalyzer
     end
     
     alias :=~ :matches
-    
-    def anonymize_value(value, capture_definition)
-      if capture_definition[:anonymize].respond_to?(:call)
-        capture_definition[:anonymize].call(value, capture_definition)
+
+    def match_for(line, request, lineno = nil, parser = nil)
+      if match_info = matches(line, lineno, parser)
+        convert_captured_values(match_info[:captures], request)
       else
-        case capture_definition[:anonymize]
-        when nil;   value
-        when false; value
-        when true;  '***'
-        when :slightly; anonymize_slightly(value, capture_definition)
-        else 
-          method_name = "anonymizer_for_#{capture_definition[:anonymize]}".to_sym
-          self.respond_to?(method_name) ? self.send(method_name, value, capture_definition) : '***'
-        end
+        false
       end
     end
-    
-    def anonymize_slightly(value, capture_definition)  
-      case capture_definition[:type]
-      when :integer
-        (value.to_i * (0.8 + rand * 0.4)).to_i
-      when :double
-        (value.to_f * (0.8 + rand * 0.4)).to_f          
-      when :msec
-        (value.to_i * (0.8 + rand * 0.4)).to_i
-      when :sec
-        (value.to_f * (0.8 + rand * 0.4)).to_f
-      when :timestamp
-        (DateTime.parse(value) + (rand(100) - 50)).to_s
-      else
-        puts "Cannot anonymize #{capture_definition[:type].inspect} slightly, using ***"
-        '***'
-      end  
-    end
 
-    # Anonymize a log line
-    def anonymize(line, options = {})
-      if self.teaser.nil? || self.teaser =~ line
-        if self.regexp =~ line
-          pos_adjustment = 0
-          captures.each_with_index do |capture, index|
-            unless $~[index + 1].nil?
-              anonymized_value = anonymize_value($~[index + 1], capture).to_s
-              line[($~.begin(index + 1) + pos_adjustment)...($~.end(index + 1) + pos_adjustment)] = anonymized_value
-              pos_adjustment += anonymized_value.length - $~[index + 1].length                    
-            end
-          end
-          line
-        elsif self.teaser.nil?
-          nil
-        else
-          options[:discard_teaser_lines] ? "" : line
-        end
-      else
-        nil
+    def convert_captured_values(values, request)
+      value_hash = {}
+      captures.each_with_index do |capture, index|
+        value_hash[capture[:name]] ||= request.convert_value(values[index], capture)
       end
+      return value_hash
     end
+
   end
   
 end

data/lib/request_log_analyzer/log_processor.rb

@@ -6,12 +6,9 @@ module RequestLogAnalyzer
   # contents of the line, remain it intact or remove it altogether, based on the current
   # file format
   #
-  # Currently, two processors are supported, :strip and :anonymize. 
+  # Currently, one processors is supported:
   #  * :strip will remove all irrelevent lines (according to the file format) from the 
   #    sources. A compact, information packed log will remain/.
-  #  * :anonymize will anonymize sensitive information from the lines according to the 
-  #    anonymization rules in the file format. The result can be passed to third parties
-  #    without privacy concerns.
   #
   class LogProcessor
     
@@ -21,8 +18,8 @@ module RequestLogAnalyzer
     attr_accessor :output_file
    
     # Builds a logprocessor instance from the arguments given on the command line
-    # <tt>command</tt> The command hat was used to start the log processor. This can either be 
-    #   :strip or :anonymize. This will set the processing mode.
+    # <tt>command</tt> The command hat was used to start the log processor. This will set the 
+    #    processing mode. Currently, only :strip is supported.
     # <tt>arguments</tt> The parsed command line arguments (a CommandLine::Arguments instance)
     def self.build(command, arguments)
       
@@ -43,7 +40,7 @@ module RequestLogAnalyzer
     
     # Initializes a new LogProcessor instance.
     # <tt>format</tt> The file format to use (e.g. :rails).
-    # <tt>mode</tt> The processing mode (:anonymize or :strip)
+    # <tt>mode</tt> The processing mode
     # <tt>options</tt> A hash with options to take into account
     def initialize(format, mode, options = {})
       @options = options
@@ -61,12 +58,11 @@ module RequestLogAnalyzer
     end
 
     # Processes an input stream by iteration over each line and processing it according to
-    # the current operation mode (:strip, :anonymize)
+    # the current operation mode
     # <tt>io</tt> The IO instance to process.
     def process_io(io)
       case mode
         when :strip;     io.each_line { |line| @output << strip_line(line) }
-        when :anonymize; io.each_line { |line| @output << anonymize_line(line) }
       end
     end
     
@@ -77,22 +73,6 @@ module RequestLogAnalyzer
       file_format.line_definitions.any? { |name, definition| definition =~ line } ? line : ""
     end
 
-    # Returns an anonymized version of the provided line. This can be a copy of the line it self, 
-    # an empty string or a string in which some substrings are substituted for anonymized values.
-    # <tt>line</tt> The line to anonymize
-    def anonymize_line(line)
-      anonymized_line = nil
-      file_format.line_definitions.detect { |name, definition| anonymized_line = definition.anonymize(line, options) }
-      
-      if anonymized_line
-        return anonymized_line
-      elsif options[:keep_junk_lines]
-        return line
-      else
-        return ""
-      end
-    end
-
     # Runs the log processing by setting up the output stream and iterating over all the
     # input sources. Input sources can either be filenames (String instances) or IO streams 
     # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable.

data/lib/request_log_analyzer/request.rb

@@ -9,7 +9,45 @@ module RequestLogAnalyzer
   # Request#every(field_name) returns all values corresponding to the given field name as array.
   class Request
   
+    module Converters
+      
+      def convert_value(value, capture_definition)
+        custom_converter_method = "convert_#{capture_definition[:type]}".to_sym
+        if respond_to?(custom_converter_method) 
+          send(custom_converter_method, value, capture_definition)
+        elsif !value.nil? 
+          case capture_definition[:type]
+          when :decimal;  value.to_f
+          when :float;    value.to_f
+          when :double;   value.to_f
+          when :integer;  value.to_i
+          when :int;      value.to_i
+          when :symbol;   value.to_sym
+          else;           value.to_s
+          end
+        else 
+          nil
+        end
+      end
+      
+      # Slow default method to parse timestamps
+      def convert_timestamp(value, capture_definition)
+        DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i unless value.nil?
+      end
+      
+      def convert_duration(value, capture_definition)
+        if value.nil?
+          nil
+        elsif capture_definition[:unit] == :msec
+          value.to_f / 1000.0      
+        else
+          value.to_f
+        end
+      end
+    end
+  
     include RequestLogAnalyzer::FileFormat::Awareness
+    include Converters
   
     attr_reader :lines
     attr_reader :attributes
@@ -32,12 +70,22 @@ module RequestLogAnalyzer
      
     # Adds another line to the request.
     # The line should be provides as a hash of the fields parsed from the line.
-    def add_parsed_line (request_info_hash)
-      @lines << request_info_hash
-      @attributes = request_info_hash.merge(@attributes)
+    def add_parsed_line (parsed_line)
+      value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
+      value_hash[:line_type] = parsed_line[:line_definition].name
+      value_hash[:lineno] = parsed_line[:lineno]
+      add_line_hash(value_hash)
+    end
+    
+    def add_line_hash(value_hash)
+      @lines << value_hash
+      @attributes = value_hash.merge(@attributes)      
     end
     
-    alias :<< :add_parsed_line
+    
+    def <<(hash)
+      hash[:line_definition] ? add_parsed_line(hash) : add_line_hash(hash)
+    end
     
     # Checks whether the given line type was parsed from the log file for this request
     def has_line_type?(line_type)
@@ -81,6 +129,10 @@ module RequestLogAnalyzer
       header_found && footer_found      
     end
     
+    # This function is called before a Requests is yielded.
+    def validate
+    end
+    
     # Returns the first timestamp encountered in a request. 
     def timestamp
       first(:timestamp)

data/lib/request_log_analyzer/source/log_parser.rb

@@ -131,7 +131,6 @@ module RequestLogAnalyzer::Source
       if header_line?(request_data)
         unless @current_request.nil?
           if options[:assume_correct_order]
-            @parsed_requests += 1
             handle_request(@current_request, &block) #yield @current_request
             @current_request = @file_format.create_request(request_data)
           else
@@ -146,7 +145,6 @@ module RequestLogAnalyzer::Source
         unless @current_request.nil?
           @current_request << request_data
           if footer_line?(request_data)
-            @parsed_requests += 1
             handle_request(@current_request, &block) # yield @current_request
             @current_request = nil 
           end
@@ -161,18 +159,19 @@ module RequestLogAnalyzer::Source
     # The default controller will send the request to every running aggegator.
     def handle_request(request, &block)
       @parsed_requests += 1
+      request.validate
       accepted = block_given? ? yield(request) : true
       @skipped_requests += 1 if not accepted
     end    
 
     # Checks whether a given line hash is a header line.
     def header_line?(hash)
-      file_format.line_definitions[hash[:line_type]].header
+      hash[:line_definition].header
     end
 
     # Checks whether a given line hash is a footer line.    
     def footer_line?(hash)
-      file_format.line_definitions[hash[:line_type]].footer  
+      hash[:line_definition].footer
     end 
   end
 

data/lib/request_log_analyzer/tracker/{category.rb → frequency.rb}

@@ -1,6 +1,6 @@
 module RequestLogAnalyzer::Tracker
   
-  # Catagorize requests.
+  # Catagorize requests by frequency.
   # Count and analyze requests for a specific attribute 
   #
   # Accepts the following options:
@@ -19,7 +19,7 @@ module RequestLogAnalyzer::Tracker
   #  PUT    |  13685 hits (28.4%) |░░░░░░░░░░░
   #  POST   |  11662 hits (24.2%) |░░░░░░░░░
   #  DELETE |    512 hits (1.1%)  |
-  class Category < Base
+  class Frequency < Base
 
     attr_reader :categories
 

data/spec/controller_spec.rb

@@ -4,42 +4,61 @@ describe RequestLogAnalyzer::Controller do
 
   include RequestLogAnalyzerSpecHelper
 
-  # it "should include the file format module" do
-  #   controller = RequestLogAnalyzer::Controller.new(:rails)
-  #   (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
-  # end
-
-  it "should call the aggregators when run" do
+  it "should use a custom output generator correctly" do
     
-    mock_output = mock('output')
-    mock_output.stub!(:io).and_return($stdout)
+    mock_output = mock('RequestLogAnalyzer::Output::Base')
+    mock_output.stub!(:io).and_return(mock_io)
     mock_output.should_receive(:header)
     mock_output.should_receive(:footer)
+
+    file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    source      = RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => log_fixture(:rails_1x))  
+    controller  = RequestLogAnalyzer::Controller.new(source, :output => mock_output)
+
+    controller.run!
+  end
+
+  it "should call aggregators correctly when run" do
     
     file_format = RequestLogAnalyzer::FileFormat.load(:rails)
     source      = RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => log_fixture(:rails_1x))  
     controller  = RequestLogAnalyzer::Controller.new(source, :output => mock_output)
     
-    mock_aggregator = mock('aggregator')
+    mock_aggregator = mock('RequestLogAnalyzer::Aggregator::Base')
     mock_aggregator.should_receive(:prepare).once.ordered
     mock_aggregator.should_receive(:aggregate).with(an_instance_of(file_format.class::Request)).at_least(:twice).ordered
     mock_aggregator.should_receive(:finalize).once.ordered
     mock_aggregator.should_receive(:report).once.ordered
+  
+    controller.aggregators << mock_aggregator
+    controller.run!
+  end
+  
+  it "should call filters when run" do
+    file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    source      = RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => log_fixture(:rails_1x))  
+    controller  = RequestLogAnalyzer::Controller.new(source, :output => mock_output)
     
-    another_mock_aggregator = mock('another aggregator')
-    another_mock_aggregator.should_receive(:prepare).once.ordered
-    another_mock_aggregator.should_receive(:aggregate).with(an_instance_of(file_format.class::Request)).at_least(:twice).ordered
-    another_mock_aggregator.should_receive(:finalize).once.ordered  
-    another_mock_aggregator.should_receive(:report).once.ordered  
-
-    controller.aggregators << mock_aggregator << another_mock_aggregator
+    mock_filter = mock('RequestLogAnalyzer::Filter::Base')
+    mock_filter.should_receive(:prepare).once.ordered
+    mock_filter.should_receive(:filter).at_least(:twice)
+    
+    controller.should_not_receive(:aggregate_request)
+    
+    controller.filters << mock_filter
     controller.run!
   end
   
-  it "should run well from the command line" do
-    temp_file = "#{File.dirname(__FILE__)}/fixtures/temp.txt"
-    system("#{File.dirname(__FILE__)}/../bin/request-log-analyzer #{log_fixture(:rails_1x)} > #{temp_file}").should be_true
+  it "should run well from the command line with the most important features" do
+    
+    temp_file = "#{File.dirname(__FILE__)}/fixtures/report.txt"
+    temp_db   = "#{File.dirname(__FILE__)}/fixtures/output.db"
+    binary = "#{File.dirname(__FILE__)}/../bin/request-log-analyzer"
+  
+    system("#{binary} #{log_fixture(:rails_1x)} --database #{temp_db} --select Controller PeopleController --file #{temp_file} > /dev/null").should be_true
+  
     File.unlink(temp_file)
+    File.unlink(temp_db)    
   end
   
 end

data/spec/file_format_spec.rb

@@ -50,13 +50,13 @@ describe RequestLogAnalyzer::FileFormat, :format_definition do
       line.first_test :regexp => /test/, :captures => []
     end
     
+
     @second_file_format.format_definition do |line|
       line.second_test :regexp => /test/, :captures => []
     end
 
+    @first_file_format.line_definer.should_not eql(@second_file_format.line_definer)
     @first_file_format.new.should have(1).line_definitions    
-    @first_file_format.new.line_definitions[:first_test].should_not be_nil
-    @second_file_format.new.should have(1).line_definitions        
     @second_file_format.new.line_definitions[:second_test].should_not be_nil
   end  
 end

data/spec/file_formats/spec_format.rb

@@ -4,24 +4,31 @@ class SpecFormat < RequestLogAnalyzer::FileFormat::Base
     line.header = true
     line.teaser = /processing /
     line.regexp = /processing request (\d+)/
-    line.captures = [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]    
+    line.captures = [{ :name => :request_no, :type => :integer }]    
   end
   
   format_definition.test do |line|
     line.teaser = /testing /
-    line.regexp = /testing is (\w+)/
-    line.captures = [{ :name => :test_capture, :type => :string, :anonymize => true}]
+    line.regexp = /testing is (\w+)(?: in (\d+\.\d+)ms)?/
+    line.captures = [{ :name => :test_capture, :type => :test_type },
+                     { :name => :duration, :type => :duration, :unit => :msec }]
   end
   
   format_definition.last do |line|
     line.footer = true
     line.teaser = /finishing /
     line.regexp = /finishing request (\d+)/
-    line.captures = [{ :name => :request_no, :type => :integer}]
+    line.captures = [{ :name => :request_no, :type => :integer }]
   end
   
   report do |analyze|
-    analyze.category :test_capture, :title => 'What is testing exactly?'
+    analyze.frequency :test_capture, :title => 'What is testing exactly?'
+  end
+  
+  class Request < RequestLogAnalyzer::Request
+    def convert_test_type(value, definition)
+      "Testing is #{value}"
+    end
   end
   
 end

data/spec/filter_spec.rb

@@ -134,11 +134,11 @@ describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
   end  
 end
 
-describe RequestLogAnalyzer::Filter::Anonimize, 'anonimize request' do
+describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
   include RequestLogAnalyzerSpecHelper
 
   before(:each) do
-    @filter = RequestLogAnalyzer::Filter::Anonimize.new(spec_format)
+    @filter = RequestLogAnalyzer::Filter::Anonymize.new(spec_format)
     @filter.prepare
   end
   
@@ -150,7 +150,7 @@ describe RequestLogAnalyzer::Filter::Anonimize, 'anonimize request' do
     @filter.filter(request(:url => 'https://test.mysite.com/employees'))[:url].should eql('http://example.com/employees')
   end
 
-  it "should anonimize url" do
+  it "should fuzz durations" do
     @filter.filter(request(:duration => 100))[:duration].should_not eql(100)
   end
   

data/spec/line_definition_spec.rb

@@ -22,103 +22,36 @@ describe RequestLogAnalyzer::LineDefinition, :parsing do
     (@line_definition =~ "Testing LineDefinition, tries: 123").should be_kind_of(Hash)
   end
   
-  it "should return a hash with all captures set" do
+  it "should return a hash with :captures set to an array" do
     hash = @line_definition.matches("Testing LineDefinition, tries: 123")
-    hash[:what].should == "LineDefinition"
-    hash[:tries].should == 123
+    hash[:captures][0].should == "LineDefinition"
+    hash[:captures][1].should == "123"
   end
   
-  it "should return a hash with :line_type set" do
-     @line_definition.matches("Testing LineDefinition, tries: 123")[:line_type].should == :test
-   end
-end
-
-describe RequestLogAnalyzer::LineDefinition, :anonymizing_basics do
-  before(:each) do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-      :teaser   => /Anonymize /,
-      :regexp   => /Anonymize (\w+)!/,
-      :captures => [{ :name => :what, :type => :string }]
-    })
-  end 
-  
-  it "should return nil if the teaser does not match" do
-    @line_definition.anonymize("Nonsense").should be_nil
-  end
-  
-  it "should return nil if no teaser exists and the regexp doesn't match" do
-    line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize!/, :captures => []})
-
-    line_definition.anonymize('nonsense').should be_nil
-  end
-  
-  it "should return itself if only the teaser matches" do
-    @line_definition.anonymize("Anonymize 456").should == "Anonymize 456"
-  end  
-  
-  it "should return an empty string if the teaser matches and discard_teaser_lines is set" do
-    @line_definition.anonymize("Anonymize 456", :discard_teaser_lines => true).should == ""
+  it "should return a hash with :line_definition set" do
+    @line_definition.matches("Testing LineDefinition, tries: 123")[:line_definition].should == @line_definition
   end
-  
-  it "should return a string if the line matches" do
-    @line_definition.anonymize("Anonymize anonymizing!").should be_kind_of(String)
-  end
-  
-  it "should not anonymize :what" do
-    @line_definition.anonymize("Anonymize anonymizing!").should == "Anonymize anonymizing!"
-  end  
 end
 
-describe RequestLogAnalyzer::LineDefinition, :anonymizing_specifics do
-
-  it "should anonymize completely if anonymize is true" do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => true }]})
+describe RequestLogAnalyzer::LineDefinition, :converting do
 
-    @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
-  end    
-    
-  it "should anonymize a URL" do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :url }]})
-    
-    @line_definition.anonymize("Anonymize https://www.not-anonymous.com/path/to/file.html!").should == "Anonymize http://example.com/path/to/file.html!"
-  end
-  
-  it "should anonymize an IP address" do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :ip }]})
-    
-    @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize 127.0.0.1!"
-  end 
+  include RequestLogAnalyzerSpecHelper
   
-  it "should anonymize completely if the anonymizer is unknown" do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :string, :anonymize => :unknown }]})
-
-    @line_definition.anonymize("Anonymize 1.2.3.4!").should == "Anonymize ***!"
+  before(:each) do
+    @file_format = spec_format
+    @request = @file_format.create_request
   end
   
-  it "should anonymize an integer slightly" do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
-
-    @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
+  it "should convert captures to a hash of converted values" do
+    hash = @file_format.line_definitions[:first].convert_captured_values(["456"], @request)
+    hash[:request_no].should == 456
   end
-  
-  it "should anonymize an integer slightly" do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :integer, :anonymize => :slightly }]})
 
-    @line_definition.anonymize("Anonymize 1234!").should =~ /Anonymize \d{3,4}\!/
+  it "should convert captures to a hash" do
+    hash = @file_format.line_definitions[:test].convert_captured_values(["willem", nil], @request)
+    hash[:test_capture].should == 'Testing is willem'
+    hash[:duration].should be_nil
   end
-  
-  it "should anonymize an double slightly" do
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
-          :regexp => /Anonymize (.+)!/, :captures => [{ :name => :what, :type => :double, :anonymize => :slightly }]})
 
-    @line_definition.anonymize("Anonymize 1.3!").should =~ /Anonymize 1\.\d+\!/
-  end
-   
+  
 end

data/spec/log_parser_spec.rb

@@ -28,10 +28,9 @@ describe RequestLogAnalyzer::Source::LogParser, :requests do
   
   it "should parse all request values when spanned over multiple files" do
     @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)]) do |request|
-      request.lines.should have(4).items
-      
+      request.lines.should have(4).items      
       request[:request_no].should == 1
-      request[:test_capture].should == "amazing"      
+      request[:test_capture].should == "Testing is amazing" # Note the custom converter     
     end
   end  
   

data/spec/log_processor_spec.rb

@@ -1,44 +1,6 @@
 require File.dirname(__FILE__) + '/spec_helper'
 require 'request_log_analyzer/log_processor'
 
-describe RequestLogAnalyzer::LogProcessor, 'anonymization' do
-
-  include RequestLogAnalyzerSpecHelper
-  
-  before(:each) do
-    @log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {})
-    @alternate_log_anonymizer = RequestLogAnalyzer::LogProcessor.new(spec_format, :anonymize, {:keep_junk_lines => true, :discard_teaser_lines => true})
-  end
-  
-  it "should keep a junk line if :keep_junk_lines is true" do
-    @alternate_log_anonymizer.anonymize_line("junk line\n").should == "junk line\n"
-  end
-  
-  it "should remove a junk line" do
-    @log_anonymizer.anonymize_line("junk line\n").should be_empty
-  end
-
-  it "should keep a teaser line intact" do
-    @log_anonymizer.anonymize_line("processing 1234\n").should == "processing 1234\n"
-  end
-  
-  it "should discard a teaser line if discard_teaser_line is true" do
-    @alternate_log_anonymizer.anonymize_line("processing 1234\n").should be_empty
-  end
-  
-  it "should keep a matching line intact if no anonymizing is declared" do
-    @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should == "finishing request 130\n"
-  end  
-
-  it "should anonymize values completely if requested" do
-    @alternate_log_anonymizer.anonymize_line("testing is great\n").should == "testing is ***\n"
-  end  
-  
-  it "should anonymize values slightly if requested" do
-    @alternate_log_anonymizer.anonymize_line("finishing request 130\n").should =~ /^finishing request 1\d\d\n$/
-  end
-end
-
 describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
 
   include RequestLogAnalyzerSpecHelper

data/spec/merb_format_spec.rb

@@ -29,7 +29,7 @@ describe RequestLogAnalyzer::Source::LogParser, :merb do
     @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
     
     request.should be_completed
-    #request[:timestamp].should == DateTime.parse('Fri Aug 29 11:10:23 +0200 2008') # FIX ME
+    request[:timestamp].should == 20080829111023 # 'Fri Aug 29 11:10:23 +0200 2008'
     request[:dispatch_time].should == 0.243424
     request[:after_filters_time].should == 6.9e-05
     request[:before_filters_time].should == 0.213213

data/spec/rails_format_spec.rb

@@ -80,6 +80,7 @@ describe "RequestLogAnalyzer::FileFormat::RailsDevelopment - Rails with developm
   
   before(:each) do
     @file_format = RequestLogAnalyzer::FileFormat.load(:rails_development)
+    @request = @file_format.create_request
   end
   
   it "should have a valid language definitions" do
@@ -87,33 +88,33 @@ describe "RequestLogAnalyzer::FileFormat::RailsDevelopment - Rails with developm
   end
   
   it "should parse a rendered line" do
-    info = @file_format.line_definitions[:rendered].matches("Rendered layouts/_footer (2.9ms)")
+    info = @file_format.line_definitions[:rendered].match_for("Rendered layouts/_footer (2.9ms)", @request)
     info[:render_file].should == 'layouts/_footer'
     info[:render_duration].should == 0.0029
   end
   
   it "should parse a query executed line with colors" do
-    info = @file_format.line_definitions[:query_executed].matches(" User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) ")
+    info = @file_format.line_definitions[:query_executed].match_for(" User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) ", @request)
     info[:query_class].should == 'User'
     info[:query_duration].should == 0.0004
     info[:query_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 18205844)'
   end  
   
   it "should parse a query executed line without colors" do
-    info = @file_format.line_definitions[:query_executed].matches(" User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) ")
+    info = @file_format.line_definitions[:query_executed].match_for(" User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) ", @request)
     info[:query_class].should == 'User'
     info[:query_duration].should == 0.0004
     info[:query_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 18205844)'    
   end  
   
   it "should parse a cached query line with colors" do
-    info = @file_format.line_definitions[:query_cached].matches(' CACHE (0.0ms)   SELECT * FROM `users` WHERE (`users`.`id` = 0) ')
+    info = @file_format.line_definitions[:query_cached].match_for(' CACHE (0.0ms)   SELECT * FROM `users` WHERE (`users`.`id` = 0) ', @request)
     info[:cached_duration].should == 0.0
     info[:cached_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 0)'   
   end
   
   it "should parse a cached query line without colors" do
-    info = @file_format.line_definitions[:query_cached].matches(' CACHE (0.0ms)   SELECT * FROM `users` WHERE (`users`.`id` = 0) ')
+    info = @file_format.line_definitions[:query_cached].match_for(' CACHE (0.0ms)   SELECT * FROM `users` WHERE (`users`.`id` = 0) ', @request)
     info[:cached_duration].should == 0.0
     info[:cached_sql].should == 'SELECT * FROM `users` WHERE (`users`.`id` = 0)'   
   end  

data/spec/spec_helper.rb

@@ -28,6 +28,22 @@ module RequestLogAnalyzerSpecHelper
       format.create_request(fields)      
     end
   end
+
+  def mock_io
+    mio = mock('IO')
+    mio.stub!(:print)
+    mio.stub!(:puts)    
+    mio.stub!(:write)
+    return mio
+  end
+  
+  def mock_output
+    output = mock('RequestLogAnalyzer::Output')
+    output.stub!(:header)
+    output.stub!(:footer)    
+    output.stub!(:io).and_return(mock_io)
+    return output
+  end
   
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.1.1
+  version: 1.1.2
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-21 00:00:00 -08:00
+date: 2009-01-24 00:00:00 -08:00
 default_executable: request-log-analyzer
 dependencies: []
 
@@ -49,7 +49,7 @@ files:
 - lib/request_log_analyzer/file_format/rails_development.rb
 - lib/request_log_analyzer/filter
 - lib/request_log_analyzer/filter.rb
-- lib/request_log_analyzer/filter/anonimize.rb
+- lib/request_log_analyzer/filter/anonymize.rb
 - lib/request_log_analyzer/filter/field.rb
 - lib/request_log_analyzer/filter/timespan.rb
 - lib/request_log_analyzer/line_definition.rb
@@ -64,8 +64,8 @@ files:
 - lib/request_log_analyzer/source/log_parser.rb
 - lib/request_log_analyzer/tracker
 - lib/request_log_analyzer/tracker.rb
-- lib/request_log_analyzer/tracker/category.rb
 - lib/request_log_analyzer/tracker/duration.rb
+- lib/request_log_analyzer/tracker/frequency.rb
 - lib/request_log_analyzer/tracker/hourly_spread.rb
 - lib/request_log_analyzer/tracker/timespan.rb
 - spec