request-log-analyzer 1.0.2

data/lib/request_log_analyzer/filter/base.rb

@@ -0,0 +1,29 @@
+module RequestLogAnalyzer
+  module Filter
+    # Base filter class used to filter input requests.
+    # All filters should interit from this base.
+    class Base
+      
+      include RequestLogAnalyzer::FileFormat::Awareness
+      
+      attr_reader :log_parser
+      attr_reader :options
+      
+      # Initializer
+      # <tt>format</tt> The file format
+      # <tt>options</tt> Are passed to the filters.
+      def initialize(format, options = {})
+        @options    = options
+        register_file_format(format)
+      end
+      
+      def prepare
+      end
+      
+      def filter(request)
+        return nil unless request
+        return request
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/filter/field.rb

@@ -0,0 +1,36 @@
+module RequestLogAnalyzer::Filter
+  
+  # Filter to select or reject a specific field
+  # Options
+  # * <tt>:mode</tt> :reject or :accept.
+  # * <tt>:field</tt> Specific field to accept or reject.
+  # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
+  class Field < Base
+   
+    attr_reader :field, :value, :mode
+   
+    def prepare
+      @mode = (@options[:mode] || :accept).to_sym
+      @field = @options[:field].to_sym
+      
+      # Convert the timestamp to the correct formats for quick timestamp comparisons
+      if @options[:value].kind_of?(String) && @options[:value][0, 1] == '/' && @options[:value][-1, 1] == '/'
+        @value = Regexp.new(@options[:value][1..-2])
+      else
+        @value = @options[:value] # TODO: convert value?
+      end
+    end
+    
+    def filter(request)
+      return nil unless request
+      
+      found_field = request.every(@field).any? { |value| @value === value }
+      
+      return nil if !found_field && @mode == :select
+      return nil if found_field && @mode == :reject
+      
+      return request
+    end 
+  end
+  
+end

data/lib/request_log_analyzer/filter/timespan.rb

@@ -0,0 +1,32 @@
+module RequestLogAnalyzer::Filter
+  
+  # Reject all requests not in given timespan
+  # Options
+  # * <tt>:after</tt> Only keep requests after this DateTime.
+  # * <tt>:before</tt> Only keep requests before this DateTime.
+  class Timespan < Base
+   
+    attr_reader :before, :after
+   
+    def prepare
+      # Convert the timestamp to the correct formats for quick timestamp comparisons
+      @after  = @options[:after].strftime('%Y%m%d%H%M%S').to_i  if options[:after]     
+      @before = @options[:before].strftime('%Y%m%d%H%M%S').to_i if options[:before]
+    end
+    
+    def filter(request)
+      return nil unless request
+      
+      if @after && @before && request.timestamp <= @before && @after <= request.timestamp
+        return request
+      elsif @after && @before.nil? && @after <= request.timestamp
+        return request
+      elsif @before && @after.nil? && request.timestamp <= @before 
+        return request
+      end
+
+      return nil
+    end 
+  end
+  
+end

data/lib/request_log_analyzer/line_definition.rb

@@ -0,0 +1,159 @@
+module RequestLogAnalyzer
+  
+  module Anonymizers
+    def anonymizer_for_ip(value, capture_definition)
+      '127.0.0.1'
+    end
+
+    def anonymizer_for_url(value, capture_definition)
+      value.sub(/^https?\:\/\/[A-z0-9\.-]+\//, "http://example.com/")
+    end
+  end  
+  
+  # The line definition class is used to specify what lines should be parsed from the log file.
+  # It contains functionality to match a line against the definition and parse the information
+  # from this line. This is used by the LogParser class when parsing a log file..
+  class LineDefinition
+
+    include RequestLogAnalyzer::Anonymizers
+
+    class Definer
+      
+      attr_accessor :line_definitions
+      
+      def initialize
+        @line_definitions = {}
+      end
+      
+      def method_missing(name, *args, &block)
+        if block_given?
+          @line_definitions[name] = RequestLogAnalyzer::LineDefinition.define(name, &block)
+        else
+          @line_definitions[name] = RequestLogAnalyzer::LineDefinition.new(name, args.first)
+        end
+      end
+    end
+
+    attr_reader :name
+    attr_accessor :teaser, :regexp, :captures
+    attr_accessor :header, :footer
+    
+    # Initializes the LineDefinition instance with a hash containing the different elements of
+    # the definition.
+    def initialize(name, definition = {})
+      @name     = name
+      @captures = []
+      definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
+    end
+    
+    def self.define(name, &block)
+      definition = self.new(name)
+      yield(definition) if block_given?
+      return definition
+    end
+    
+    # Converts a parsed value (String) to the desired value using some heuristics.
+    def convert_value(value, type)
+      case type
+      when :integer;   value.to_i
+      when :float;     value.to_f
+      when :decimal;   value.to_f          
+      when :symbol;    value.to_sym
+      when :sec;       value.to_f
+      when :msec;      value.to_f / 1000
+      when :timestamp; value.gsub(/[^0-9]/,'')[0..13].to_i # Retrieve with: DateTime.parse(value, '%Y%m%d%H%M%S')
+      else value
+      end
+    end
+    
+    # Checks whether a given line matches this definition. 
+    # It will return false if a line does not match. If the line matches, a hash is returned
+    # with all the fields parsed from that line as content.
+    # If the line definition has a teaser-check, a :teaser_check_failed warning will be emitted
+    # if this teaser-check is passed, but the full regular exprssion does not ,atch.
+    def matches(line, lineno = nil, parser = nil)
+      if @teaser.nil? || @teaser =~ line
+        if match_data = line.match(@regexp)
+          request_info = { :line_type => name, :lineno => lineno }
+          
+          captures.each_with_index do |capture, index|
+            next if capture == :ignore
+
+            if match_data.captures[index]
+              request_info[capture[:name]] = convert_value(match_data.captures[index], capture[:type])
+            end
+
+          end
+          return request_info
+        else
+          if @teaser && parser
+            parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")
+          end
+          return false
+        end
+      else
+        return false
+      end
+    end
+    
+    alias :=~ :matches
+    
+    def anonymize_value(value, capture_definition)
+      if capture_definition[:anonymize].respond_to?(:call)
+        capture_definition[:anonymize].call(value, capture_definition)
+      else
+        case capture_definition[:anonymize]
+        when nil;   value
+        when false; value
+        when true;  '***'
+        when :slightly; anonymize_slightly(value, capture_definition)
+        else 
+          method_name = "anonymizer_for_#{capture_definition[:anonymize]}".to_sym
+          self.respond_to?(method_name) ? self.send(method_name, value, capture_definition) : '***'
+        end
+      end
+    end
+    
+    def anonymize_slightly(value, capture_definition)  
+      case capture_definition[:type]
+      when :integer
+        (value.to_i * (0.8 + rand * 0.4)).to_i
+      when :double
+        (value.to_f * (0.8 + rand * 0.4)).to_f          
+      when :msec
+        (value.to_i * (0.8 + rand * 0.4)).to_i
+      when :sec
+        (value.to_f * (0.8 + rand * 0.4)).to_f
+      when :timestamp
+        (DateTime.parse(value) + (rand(100) - 50)).to_s
+      else
+        puts "Cannot anonymize #{capture_definition[:type].inspect} slightly, using ***"
+        '***'
+      end  
+    end
+
+    # Anonymize a log line
+    def anonymize(line, options = {})
+      if self.teaser.nil? || self.teaser =~ line
+        if self.regexp =~ line
+          pos_adjustment = 0
+          captures.each_with_index do |capture, index|
+            unless $~[index + 1].nil?
+              anonymized_value = anonymize_value($~[index + 1], capture).to_s
+              line[($~.begin(index + 1) + pos_adjustment)...($~.end(index + 1) + pos_adjustment)] = anonymized_value
+              pos_adjustment += anonymized_value.length - $~[index + 1].length                    
+            end
+          end
+          line
+        elsif self.teaser.nil?
+          nil
+        else
+          options[:discard_teaser_lines] ? "" : line
+        end
+      else
+        nil
+      end
+    end
+  end
+  
+end

data/lib/request_log_analyzer/log_parser.rb

@@ -0,0 +1,173 @@
+module RequestLogAnalyzer
+  
+  # The LogParser class reads log data from a given source and uses a file format definition
+  # to parse all relevent information about requests from the file. A FileFormat module should 
+  # be provided that contains the definitions of the lines that occur in the log data.
+  #
+  # De order in which lines occur is used to combine lines to a single request. If these lines 
+  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
+  # the log file simultaneously by different mongrel processes. This problem is detected by the 
+  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
+  # occurs.
+  class LogParser
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    # A hash of options
+    attr_reader :options
+    
+    # The current Request object that is being parsed
+    attr_reader :current_request
+    
+    # The total number of parsed lines
+    attr_reader :parsed_lines
+    
+    # The total number of parsed requests.
+    attr_reader :parsed_requests
+    
+    # The number of skipped requests because of date constraints
+    attr_reader :skipped_requests
+    
+    # Initializes the parser instance.
+    # It will apply the language specific FileFormat module to this instance. It will use the line
+    # definitions in this module to parse any input.
+    def initialize(format, options = {})      
+      @line_definitions = {}
+      @options          = options
+      @parsed_lines     = 0
+      @parsed_requests  = 0
+      @skipped_requests = 0      
+      
+      @current_io = nil
+      
+      # install the file format module (see RequestLogAnalyzer::FileFormat)
+      # and register all the line definitions to the parser
+      self.register_file_format(format)
+    end
+    
+    # Parses a list of consequent files of the same format
+    def parse_files(files, options = {}, &block)
+      files.each { |file| parse_file(file, options, &block) }
+    end
+    
+    # Parses a file. 
+    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
+    def parse_file(file, options = {}, &block)
+      @progress_handler.call(:started, file) if @progress_handler
+      File.open(file, 'r') { |f| parse_io(f, options, &block) }
+      @progress_handler.call(:finished, file) if @progress_handler
+    end
+    
+    def parse_stream(stream, options = {}, &block)
+      parse_io(stream, options, &block)
+    end
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def parse_io(io, options = {}, &block)
+
+      # parse every line type by default
+      line_types = options[:line_types] || file_format.line_definitions.keys
+
+      # check whether all provided line types are valid
+      unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
+      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
+      
+      @current_io = io
+      @current_io.each_line do |line|
+        
+        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
+        
+        request_data = nil
+        line_types.each do |line_type|
+          line_type_definition = file_format.line_definitions[line_type]
+          break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
+        end
+
+        if request_data
+          @parsed_lines += 1
+          update_current_request(request_data, &block)
+        end
+      end
+      
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+  
+      @current_io = nil
+    end
+    
+    # Add a block to this method to install a progress handler while parsing
+    def progress=(proc)
+      @progress_handler = proc
+    end
+    
+    # Add a block to this method to install a warning handler while parsing
+    def warning=(proc)
+      @warning_handler = proc
+    end
+    
+    # This method is called by the parser if it encounteres any problems.
+    # It will call the warning handler. The default controller will pass all warnings to every
+    # aggregator that is registered and running
+    def warn(type, message)
+      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
+    end
+    
+    protected
+    
+    # Combines the different lines of a request into a single Request object. It will start a 
+    # new request when a header line is encountered en will emit the request when a footer line 
+    # is encountered.
+    #
+    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    #   any request. It will emit a :no_current_request warning.
+    # - A header line that is parsed before a request is closed by a footer line, is a sign of
+    #   an unprpertly ordered file. All data that is gathered for the request until then is 
+    #   discarded, the next request is ignored as well and a :unclosed_request warning is
+    #   emitted.
+    def update_current_request(request_data, &block)
+      if header_line?(request_data)
+        unless @current_request.nil?
+          if options[:assume_correct_order]
+            handle_request(@current_request, &block)            
+            @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
+          else
+            warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
+            @current_request = nil # remove all data that was parsed, skip next request as well.
+          end
+        else
+          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
+        end
+      else
+        unless @current_request.nil?
+          @current_request << request_data
+          if footer_line?(request_data)
+            handle_request(@current_request, &block)
+            @current_request = nil 
+          end
+        else
+          warn(:no_current_request, "Parsebale line found outside of a request!")
+        end
+      end
+    end
+    
+    # Handles the parsed request by calling the request handler.
+    # The default controller will send the request to every running aggegator.
+    def handle_request(request, &block)
+      @parsed_requests += 1
+      accepted = block_given? ? yield(request) : true
+      @skipped_requests += 1 if !accepted
+    end
+    
+    # Checks whether a given line hash is a header line.
+    def header_line?(hash)
+      file_format.line_definitions[hash[:line_type]].header
+    end
+
+    # Checks whether a given line hash is a footer line.    
+    def footer_line?(hash)
+      file_format.line_definitions[hash[:line_type]].footer  
+    end 
+  end
+end

data/lib/request_log_analyzer/log_processor.rb

@@ -0,0 +1,121 @@
+module RequestLogAnalyzer
+  
+  # The Logprocessor class is used to perform simple processing actions over log files.
+  # It will go over the log file/stream line by line, pass the line to a processor and 
+  # write the result back to the output file or stream. The processor can alter the 
+  # contents of the line, remain it intact or remove it altogether, based on the current
+  # file format
+  #
+  # Currently, two processors are supported, :strip and :anonymize. 
+  #  * :strip will remove all irrelevent lines (according to the file format) from the 
+  #    sources. A compact, information packed log will remain/.
+  #  * :anonymize will anonymize sensitive information from the lines according to the 
+  #    anonymization rules in the file format. The result can be passed to third parties
+  #    without privacy concerns.
+  #
+  class LogProcessor
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    attr_reader :mode, :options, :sources
+    attr_accessor :output_file
+   
+    # Builds a logprocessor instance from the arguments given on the command line
+    # <tt>command</tt> The command hat was used to start the log processor. This can either be 
+    #   :strip or :anonymize. This will set the processing mode.
+    # <tt>arguments</tt> The parsed command line arguments (a CommandLine::Arguments instance)
+    def self.build(command, arguments)
+      
+      options = { 
+          :discard_teaser_lines => arguments[:discard_teaser_lines], 
+          :keep_junk_lines      => arguments[:keep_junk_lines], 
+        }
+          
+      log_processor = RequestLogAnalyzer::LogProcessor.new(arguments[:format].to_sym, command, options)
+      log_processor.output_file = arguments[:output] if arguments[:output]
+
+      arguments.parameters.each do |input|
+        log_processor.sources << input
+      end
+      
+      return log_processor
+    end
+    
+    # Initializes a new LogProcessor instance.
+    # <tt>format</tt> The file format to use (e.g. :rails).
+    # <tt>mode</tt> The processing mode (:anonymize or :strip)
+    # <tt>options</tt> A hash with options to take into account
+    def initialize(format, mode, options = {})
+      @options = options
+      @mode    = mode
+      @sources = []
+      $output_file = nil
+      self.register_file_format(format)
+    end
+    
+    # Processes input files by opening it and sending the filestream to <code>process_io</code>,
+    # in which the actual processing is performed.
+    # <tt>file</tt> The file to process
+    def process_file(file)
+      File.open(file, 'r') { |file| process_io(file) }
+    end
+
+    # Processes an input stream by iteration over each line and processing it according to
+    # the current operation mode (:strip, :anonymize)
+    # <tt>io</tt> The IO instance to process.
+    def process_io(io)
+      case mode
+        when :strip;     io.each_line { |line| @output << strip_line(line) }
+        when :anonymize; io.each_line { |line| @output << anonymize_line(line) }
+      end
+    end
+    
+    # Returns the line itself if the string matches any of the line definitions. If no match is
+    # found, an empty line is returned, which will strip the line from the output.
+    # <tt>line</tt> The line to strip
+    def strip_line(line)
+      file_format.line_definitions.any? { |name, definition| definition =~ line } ? line : ""
+    end
+
+    # Returns an anonymized version of the provided line. This can be a copy of the line it self, 
+    # an empty string or a string in which some substrings are substituted for anonymized values.
+    # <tt>line</tt> The line to anonymize
+    def anonymize_line(line)
+      anonymized_line = nil
+      file_format.line_definitions.detect { |name, definition| anonymized_line = definition.anonymize(line, options) }
+      
+      if anonymized_line
+        return anonymized_line
+      elsif options[:keep_junk_lines]
+        return line
+      else
+        return ""
+      end
+    end
+
+    # Runs the log processing by setting up the output stream and iterating over all the
+    # input sources. Input sources can either be filenames (String instances) or IO streams 
+    # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable.
+    def run!
+      if @output_file.nil?
+        @output = $stdout 
+      else
+        @output = File.new(@output_file, 'a')        
+      end
+      
+      @sources.each do |source|
+        if source.kind_of?(String) && File.exist?(source)
+          process_file(source)
+        elsif source.kind_of?(IO)
+          process_io(source)
+        elsif ['-', 'STDIN'].include?(source)
+          process_io($stdin)        
+        end
+      end
+    
+    ensure
+      @output.close if @output.kind_of?(File)
+    end
+  end
+
+end