request-log-analyzer 1.1.1 → 1.1.2

data/lib/request_log_analyzer/filter/timespan.rb

@@ -22,8 +22,6 @@ module RequestLogAnalyzer::Filter
     # Returns nil otherwise
     # <tt>request</tt> Request object.
     def filter(request)
-      return nil unless request
-      
       if @after && @before && request.timestamp <= @before && @after <= request.timestamp
         return request
       elsif @after && @before.nil? && @after <= request.timestamp

data/lib/request_log_analyzer/line_definition.rb

@@ -1,22 +1,10 @@
 module RequestLogAnalyzer
   
-  module Anonymizers
-    def anonymizer_for_ip(value, capture_definition)
-      '127.0.0.1'
-    end
-
-    def anonymizer_for_url(value, capture_definition)
-      value.sub(/^https?\:\/\/[A-Za-z0-9\.-]+\//, "http://example.com/")
-    end
-  end  
-  
   # The line definition class is used to specify what lines should be parsed from the log file.
   # It contains functionality to match a line against the definition and parse the information
   # from this line. This is used by the LogParser class when parsing a log file..
   class LineDefinition
 
-    include RequestLogAnalyzer::Anonymizers
-
     class Definer
       
       attr_accessor :line_definitions
@@ -52,20 +40,6 @@ module RequestLogAnalyzer
       return definition
     end
     
-    # Converts a parsed value (String) to the desired value using some heuristics.
-    def convert_value(value, type)
-      case type
-      when :integer;   value.to_i
-      when :float;     value.to_f
-      when :decimal;   value.to_f          
-      when :symbol;    value.to_sym
-      when :sec;       value.to_f
-      when :msec;      value.to_f / 1000
-      when :timestamp; value.gsub(/[^0-9]/,'')[0..13].to_i # Retrieve with: DateTime.parse(value, '%Y%m%d%H%M%S')
-      else value
-      end
-    end
-    
     # Checks whether a given line matches this definition. 
     # It will return false if a line does not match. If the line matches, a hash is returned
     # with all the fields parsed from that line as content.
@@ -74,17 +48,7 @@ module RequestLogAnalyzer
     def matches(line, lineno = nil, parser = nil)
       if @teaser.nil? || @teaser =~ line
         if match_data = line.match(@regexp)
-          request_info = { :line_type => name, :lineno => lineno }
-          
-          captures.each_with_index do |capture, index|
-            next if capture == :ignore
-
-            if match_data.captures[index]
-              request_info[capture[:name]] = convert_value(match_data.captures[index], capture[:type])
-            end
-
-          end
-          return request_info
+          return { :line_definition => self, :lineno => lineno, :captures => match_data.captures}
         else
           if @teaser && parser
             parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")
@@ -97,63 +61,23 @@ module RequestLogAnalyzer
     end
     
     alias :=~ :matches
-    
-    def anonymize_value(value, capture_definition)
-      if capture_definition[:anonymize].respond_to?(:call)
-        capture_definition[:anonymize].call(value, capture_definition)
+
+    def match_for(line, request, lineno = nil, parser = nil)
+      if match_info = matches(line, lineno, parser)
+        convert_captured_values(match_info[:captures], request)
       else
-        case capture_definition[:anonymize]
-        when nil;   value
-        when false; value
-        when true;  '***'
-        when :slightly; anonymize_slightly(value, capture_definition)
-        else 
-          method_name = "anonymizer_for_#{capture_definition[:anonymize]}".to_sym
-          self.respond_to?(method_name) ? self.send(method_name, value, capture_definition) : '***'
-        end
+        false
       end
     end
-    
-    def anonymize_slightly(value, capture_definition)  
-      case capture_definition[:type]
-      when :integer
-        (value.to_i * (0.8 + rand * 0.4)).to_i
-      when :double
-        (value.to_f * (0.8 + rand * 0.4)).to_f          
-      when :msec
-        (value.to_i * (0.8 + rand * 0.4)).to_i
-      when :sec
-        (value.to_f * (0.8 + rand * 0.4)).to_f
-      when :timestamp
-        (DateTime.parse(value) + (rand(100) - 50)).to_s
-      else
-        puts "Cannot anonymize #{capture_definition[:type].inspect} slightly, using ***"
-        '***'
-      end  
-    end
 
-    # Anonymize a log line
-    def anonymize(line, options = {})
-      if self.teaser.nil? || self.teaser =~ line
-        if self.regexp =~ line
-          pos_adjustment = 0
-          captures.each_with_index do |capture, index|
-            unless $~[index + 1].nil?
-              anonymized_value = anonymize_value($~[index + 1], capture).to_s
-              line[($~.begin(index + 1) + pos_adjustment)...($~.end(index + 1) + pos_adjustment)] = anonymized_value
-              pos_adjustment += anonymized_value.length - $~[index + 1].length                    
-            end
-          end
-          line
-        elsif self.teaser.nil?
-          nil
-        else
-          options[:discard_teaser_lines] ? "" : line
-        end
-      else
-        nil
+    def convert_captured_values(values, request)
+      value_hash = {}
+      captures.each_with_index do |capture, index|
+        value_hash[capture[:name]] ||= request.convert_value(values[index], capture)
       end
+      return value_hash
     end
+
   end
   
 end

data/lib/request_log_analyzer/log_processor.rb

@@ -6,12 +6,9 @@ module RequestLogAnalyzer
   # contents of the line, remain it intact or remove it altogether, based on the current
   # file format
   #
-  # Currently, two processors are supported, :strip and :anonymize. 
+  # Currently, one processors is supported:
   #  * :strip will remove all irrelevent lines (according to the file format) from the 
   #    sources. A compact, information packed log will remain/.
-  #  * :anonymize will anonymize sensitive information from the lines according to the 
-  #    anonymization rules in the file format. The result can be passed to third parties
-  #    without privacy concerns.
   #
   class LogProcessor
     
@@ -21,8 +18,8 @@ module RequestLogAnalyzer
     attr_accessor :output_file
    
     # Builds a logprocessor instance from the arguments given on the command line
-    # <tt>command</tt> The command hat was used to start the log processor. This can either be 
-    #   :strip or :anonymize. This will set the processing mode.
+    # <tt>command</tt> The command hat was used to start the log processor. This will set the 
+    #    processing mode. Currently, only :strip is supported.
     # <tt>arguments</tt> The parsed command line arguments (a CommandLine::Arguments instance)
     def self.build(command, arguments)
       
@@ -43,7 +40,7 @@ module RequestLogAnalyzer
     
     # Initializes a new LogProcessor instance.
     # <tt>format</tt> The file format to use (e.g. :rails).
-    # <tt>mode</tt> The processing mode (:anonymize or :strip)
+    # <tt>mode</tt> The processing mode
     # <tt>options</tt> A hash with options to take into account
     def initialize(format, mode, options = {})
       @options = options
@@ -61,12 +58,11 @@ module RequestLogAnalyzer
     end
 
     # Processes an input stream by iteration over each line and processing it according to
-    # the current operation mode (:strip, :anonymize)
+    # the current operation mode
     # <tt>io</tt> The IO instance to process.
     def process_io(io)
       case mode
         when :strip;     io.each_line { |line| @output << strip_line(line) }
-        when :anonymize; io.each_line { |line| @output << anonymize_line(line) }
       end
     end
     
@@ -77,22 +73,6 @@ module RequestLogAnalyzer
       file_format.line_definitions.any? { |name, definition| definition =~ line } ? line : ""
     end
 
-    # Returns an anonymized version of the provided line. This can be a copy of the line it self, 
-    # an empty string or a string in which some substrings are substituted for anonymized values.
-    # <tt>line</tt> The line to anonymize
-    def anonymize_line(line)
-      anonymized_line = nil
-      file_format.line_definitions.detect { |name, definition| anonymized_line = definition.anonymize(line, options) }
-      
-      if anonymized_line
-        return anonymized_line
-      elsif options[:keep_junk_lines]
-        return line
-      else
-        return ""
-      end
-    end
-
     # Runs the log processing by setting up the output stream and iterating over all the
     # input sources. Input sources can either be filenames (String instances) or IO streams 
     # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable.

data/lib/request_log_analyzer/request.rb

@@ -9,7 +9,45 @@ module RequestLogAnalyzer
   # Request#every(field_name) returns all values corresponding to the given field name as array.
   class Request
   
+    module Converters
+      
+      def convert_value(value, capture_definition)
+        custom_converter_method = "convert_#{capture_definition[:type]}".to_sym
+        if respond_to?(custom_converter_method) 
+          send(custom_converter_method, value, capture_definition)
+        elsif !value.nil? 
+          case capture_definition[:type]
+          when :decimal;  value.to_f
+          when :float;    value.to_f
+          when :double;   value.to_f
+          when :integer;  value.to_i
+          when :int;      value.to_i
+          when :symbol;   value.to_sym
+          else;           value.to_s
+          end
+        else 
+          nil
+        end
+      end
+      
+      # Slow default method to parse timestamps
+      def convert_timestamp(value, capture_definition)
+        DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i unless value.nil?
+      end
+      
+      def convert_duration(value, capture_definition)
+        if value.nil?
+          nil
+        elsif capture_definition[:unit] == :msec
+          value.to_f / 1000.0      
+        else
+          value.to_f
+        end
+      end
+    end
+  
     include RequestLogAnalyzer::FileFormat::Awareness
+    include Converters
   
     attr_reader :lines
     attr_reader :attributes
@@ -32,12 +70,22 @@ module RequestLogAnalyzer
      
     # Adds another line to the request.
     # The line should be provides as a hash of the fields parsed from the line.
-    def add_parsed_line (request_info_hash)
-      @lines << request_info_hash
-      @attributes = request_info_hash.merge(@attributes)
+    def add_parsed_line (parsed_line)
+      value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
+      value_hash[:line_type] = parsed_line[:line_definition].name
+      value_hash[:lineno] = parsed_line[:lineno]
+      add_line_hash(value_hash)
+    end
+    
+    def add_line_hash(value_hash)
+      @lines << value_hash
+      @attributes = value_hash.merge(@attributes)      
     end
     
-    alias :<< :add_parsed_line
+    
+    def <<(hash)
+      hash[:line_definition] ? add_parsed_line(hash) : add_line_hash(hash)
+    end
     
     # Checks whether the given line type was parsed from the log file for this request
     def has_line_type?(line_type)
@@ -81,6 +129,10 @@ module RequestLogAnalyzer
       header_found && footer_found      
     end
     
+    # This function is called before a Requests is yielded.
+    def validate
+    end
+    
     # Returns the first timestamp encountered in a request. 
     def timestamp
       first(:timestamp)

data/lib/request_log_analyzer/source/log_parser.rb

@@ -131,7 +131,6 @@ module RequestLogAnalyzer::Source
       if header_line?(request_data)
         unless @current_request.nil?
           if options[:assume_correct_order]
-            @parsed_requests += 1
             handle_request(@current_request, &block) #yield @current_request
             @current_request = @file_format.create_request(request_data)
           else
@@ -146,7 +145,6 @@ module RequestLogAnalyzer::Source
         unless @current_request.nil?
           @current_request << request_data
           if footer_line?(request_data)
-            @parsed_requests += 1
             handle_request(@current_request, &block) # yield @current_request
             @current_request = nil 
           end
@@ -161,18 +159,19 @@ module RequestLogAnalyzer::Source
     # The default controller will send the request to every running aggegator.
     def handle_request(request, &block)
       @parsed_requests += 1
+      request.validate
       accepted = block_given? ? yield(request) : true
       @skipped_requests += 1 if not accepted
     end    
 
     # Checks whether a given line hash is a header line.
     def header_line?(hash)
-      file_format.line_definitions[hash[:line_type]].header
+      hash[:line_definition].header
     end
 
     # Checks whether a given line hash is a footer line.    
     def footer_line?(hash)
-      file_format.line_definitions[hash[:line_type]].footer  
+      hash[:line_definition].footer
     end 
   end
 

data/lib/request_log_analyzer/tracker/{category.rb → frequency.rb}

@@ -1,6 +1,6 @@
 module RequestLogAnalyzer::Tracker
   
-  # Catagorize requests.
+  # Catagorize requests by frequency.
   # Count and analyze requests for a specific attribute 
   #
   # Accepts the following options:
@@ -19,7 +19,7 @@ module RequestLogAnalyzer::Tracker
   #  PUT    |  13685 hits (28.4%) |░░░░░░░░░░░
   #  POST   |  11662 hits (24.2%) |░░░░░░░░░
   #  DELETE |    512 hits (1.1%)  |
-  class Category < Base
+  class Frequency < Base
 
     attr_reader :categories
 

data/spec/controller_spec.rb

@@ -4,42 +4,61 @@ describe RequestLogAnalyzer::Controller do
 
   include RequestLogAnalyzerSpecHelper
 
-  # it "should include the file format module" do
-  #   controller = RequestLogAnalyzer::Controller.new(:rails)
-  #   (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
-  # end
-
-  it "should call the aggregators when run" do
+  it "should use a custom output generator correctly" do
     
-    mock_output = mock('output')
-    mock_output.stub!(:io).and_return($stdout)
+    mock_output = mock('RequestLogAnalyzer::Output::Base')
+    mock_output.stub!(:io).and_return(mock_io)
     mock_output.should_receive(:header)
     mock_output.should_receive(:footer)
+
+    file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    source      = RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => log_fixture(:rails_1x))  
+    controller  = RequestLogAnalyzer::Controller.new(source, :output => mock_output)
+
+    controller.run!
+  end
+
+  it "should call aggregators correctly when run" do
     
     file_format = RequestLogAnalyzer::FileFormat.load(:rails)
     source      = RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => log_fixture(:rails_1x))  
     controller  = RequestLogAnalyzer::Controller.new(source, :output => mock_output)
     
-    mock_aggregator = mock('aggregator')
+    mock_aggregator = mock('RequestLogAnalyzer::Aggregator::Base')
     mock_aggregator.should_receive(:prepare).once.ordered
     mock_aggregator.should_receive(:aggregate).with(an_instance_of(file_format.class::Request)).at_least(:twice).ordered
     mock_aggregator.should_receive(:finalize).once.ordered
     mock_aggregator.should_receive(:report).once.ordered
+  
+    controller.aggregators << mock_aggregator
+    controller.run!
+  end
+  
+  it "should call filters when run" do
+    file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    source      = RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => log_fixture(:rails_1x))  
+    controller  = RequestLogAnalyzer::Controller.new(source, :output => mock_output)
     
-    another_mock_aggregator = mock('another aggregator')
-    another_mock_aggregator.should_receive(:prepare).once.ordered
-    another_mock_aggregator.should_receive(:aggregate).with(an_instance_of(file_format.class::Request)).at_least(:twice).ordered
-    another_mock_aggregator.should_receive(:finalize).once.ordered  
-    another_mock_aggregator.should_receive(:report).once.ordered  
-
-    controller.aggregators << mock_aggregator << another_mock_aggregator
+    mock_filter = mock('RequestLogAnalyzer::Filter::Base')
+    mock_filter.should_receive(:prepare).once.ordered
+    mock_filter.should_receive(:filter).at_least(:twice)
+    
+    controller.should_not_receive(:aggregate_request)
+    
+    controller.filters << mock_filter
     controller.run!
   end
   
-  it "should run well from the command line" do
-    temp_file = "#{File.dirname(__FILE__)}/fixtures/temp.txt"
-    system("#{File.dirname(__FILE__)}/../bin/request-log-analyzer #{log_fixture(:rails_1x)} > #{temp_file}").should be_true
+  it "should run well from the command line with the most important features" do
+    
+    temp_file = "#{File.dirname(__FILE__)}/fixtures/report.txt"
+    temp_db   = "#{File.dirname(__FILE__)}/fixtures/output.db"
+    binary = "#{File.dirname(__FILE__)}/../bin/request-log-analyzer"
+  
+    system("#{binary} #{log_fixture(:rails_1x)} --database #{temp_db} --select Controller PeopleController --file #{temp_file} > /dev/null").should be_true
+  
     File.unlink(temp_file)
+    File.unlink(temp_db)    
   end
   
 end

data/spec/file_format_spec.rb

@@ -50,13 +50,13 @@ describe RequestLogAnalyzer::FileFormat, :format_definition do
       line.first_test :regexp => /test/, :captures => []
     end
     
+
     @second_file_format.format_definition do |line|
       line.second_test :regexp => /test/, :captures => []
     end
 
+    @first_file_format.line_definer.should_not eql(@second_file_format.line_definer)
     @first_file_format.new.should have(1).line_definitions    
-    @first_file_format.new.line_definitions[:first_test].should_not be_nil
-    @second_file_format.new.should have(1).line_definitions        
     @second_file_format.new.line_definitions[:second_test].should_not be_nil
   end  
 end