wvanbergen-request-log-analyzer 1.0.0 → 1.0.1

data/lib/request_log_analyzer/line_definition.rb

@@ -0,0 +1,159 @@
+module RequestLogAnalyzer
+  
+  module Anonymizers
+    def anonymizer_for_ip(value, capture_definition)
+      '127.0.0.1'
+    end
+
+    def anonymizer_for_url(value, capture_definition)
+      value.sub(/^https?\:\/\/[A-z0-9\.-]+\//, "http://example.com/")
+    end
+  end  
+  
+  # The line definition class is used to specify what lines should be parsed from the log file.
+  # It contains functionality to match a line against the definition and parse the information
+  # from this line. This is used by the LogParser class when parsing a log file..
+  class LineDefinition
+
+    include RequestLogAnalyzer::Anonymizers
+
+    class Definer
+      
+      attr_accessor :line_definitions
+      
+      def initialize
+        @line_definitions = {}
+      end
+      
+      def method_missing(name, *args, &block)
+        if block_given?
+          @line_definitions[name] = RequestLogAnalyzer::LineDefinition.define(name, &block)
+        else
+          @line_definitions[name] = RequestLogAnalyzer::LineDefinition.new(name, args.first)
+        end
+      end
+    end
+
+    attr_reader :name
+    attr_accessor :teaser, :regexp, :captures
+    attr_accessor :header, :footer
+    
+    # Initializes the LineDefinition instance with a hash containing the different elements of
+    # the definition.
+    def initialize(name, definition = {})
+      @name     = name
+      @captures = []
+      definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
+    end
+    
+    def self.define(name, &block)
+      definition = self.new(name)
+      yield(definition) if block_given?
+      return definition
+    end
+    
+    # Converts a parsed value (String) to the desired value using some heuristics.
+    def convert_value(value, type)
+      case type
+      when :integer;   value.to_i
+      when :float;     value.to_f
+      when :decimal;   value.to_f          
+      when :symbol;    value.to_sym
+      when :sec;       value.to_f
+      when :msec;      value.to_f / 1000
+      when :timestamp; value.gsub(/[^0-9]/,'')[0..13].to_i # Retrieve with: DateTime.parse(value, '%Y%m%d%H%M%S')
+      else value
+      end
+    end
+    
+    # Checks whether a given line matches this definition. 
+    # It will return false if a line does not match. If the line matches, a hash is returned
+    # with all the fields parsed from that line as content.
+    # If the line definition has a teaser-check, a :teaser_check_failed warning will be emitted
+    # if this teaser-check is passed, but the full regular exprssion does not ,atch.
+    def matches(line, lineno = nil, parser = nil)
+      if @teaser.nil? || @teaser =~ line
+        if match_data = line.match(@regexp)
+          request_info = { :line_type => name, :lineno => lineno }
+          
+          captures.each_with_index do |capture, index|
+            next if capture == :ignore
+
+            if match_data.captures[index]
+              request_info[capture[:name]] = convert_value(match_data.captures[index], capture[:type])
+            end
+
+          end
+          return request_info
+        else
+          if @teaser && parser
+            parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")
+          end
+          return false
+        end
+      else
+        return false
+      end
+    end
+    
+    alias :=~ :matches
+    
+    def anonymize_value(value, capture_definition)
+      if capture_definition[:anonymize].respond_to?(:call)
+        capture_definition[:anonymize].call(value, capture_definition)
+      else
+        case capture_definition[:anonymize]
+        when nil;   value
+        when false; value
+        when true;  '***'
+        when :slightly; anonymize_slightly(value, capture_definition)
+        else 
+          method_name = "anonymizer_for_#{capture_definition[:anonymize]}".to_sym
+          self.respond_to?(method_name) ? self.send(method_name, value, capture_definition) : '***'
+        end
+      end
+    end
+    
+    def anonymize_slightly(value, capture_definition)  
+      case capture_definition[:type]
+      when :integer
+        (value.to_i * (0.8 + rand * 0.4)).to_i
+      when :double
+        (value.to_f * (0.8 + rand * 0.4)).to_f          
+      when :msec
+        (value.to_i * (0.8 + rand * 0.4)).to_i
+      when :sec
+        (value.to_f * (0.8 + rand * 0.4)).to_f
+      when :timestamp
+        (DateTime.parse(value) + (rand(100) - 50)).to_s
+      else
+        puts "Cannot anonymize #{capture_definition[:type].inspect} slightly, using ***"
+        '***'
+      end  
+    end
+
+    # Anonymize a log line
+    def anonymize(line, options = {})
+      if self.teaser.nil? || self.teaser =~ line
+        if self.regexp =~ line
+          pos_adjustment = 0
+          captures.each_with_index do |capture, index|
+            unless $~[index + 1].nil?
+              anonymized_value = anonymize_value($~[index + 1], capture).to_s
+              line[($~.begin(index + 1) + pos_adjustment)...($~.end(index + 1) + pos_adjustment)] = anonymized_value
+              pos_adjustment += anonymized_value.length - $~[index + 1].length                    
+            end
+          end
+          line
+        elsif self.teaser.nil?
+          nil
+        else
+          options[:discard_teaser_lines] ? "" : line
+        end
+      else
+        nil
+      end
+    end
+  end
+  
+end

data/lib/request_log_analyzer/log_parser.rb

@@ -0,0 +1,183 @@
+module RequestLogAnalyzer
+  
+  # The LogParser class reads log data from a given source and uses a file format definition
+  # to parse all relevent information about requests from the file.
+  #
+  # A FileFormat module should be provided that contains the definitions of the lines that 
+  # occur in the log data. The log parser can run in two modes:
+  # - In single line mode, it will emit every detected line as a separate request
+  # - In combined requests mode, it will combine the different lines from the line defintions
+  #   into one request, that will then be emitted. 
+  #
+  # The combined requests mode gives better information, but can be problematic if the log 
+  # file is unordered. This can be the case if data is written to the log file simultaneously 
+  # by different mongrel processes. This problem is detected by the parser, but the requests
+  # that are mixed up cannot be parsed. It will emit warnings when this occurs.
+  class LogParser
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    # A hash of options
+    attr_reader :options
+    
+    # The current Request object that is being parsed
+    attr_reader :current_request
+    
+    # The total number of parsed lines
+    attr_reader :parsed_lines
+    
+    # The total number of parsed requests.
+    attr_reader :parsed_requests
+    
+    # The number of skipped requests because of date constraints
+    attr_reader :skipped_requests
+    
+    # Initializes the parser instance.
+    # It will apply the language specific FileFormat module to this instance. It will use the line
+    # definitions in this module to parse any input.
+    def initialize(format, options = {})      
+      @line_definitions = {}
+      @options          = options
+      @parsed_lines     = 0
+      @parsed_requests  = 0
+      @skipped_requests = 0      
+      
+      @current_io = nil
+      
+      # install the file format module (see RequestLogAnalyzer::FileFormat)
+      # and register all the line definitions to the parser
+      self.register_file_format(format)
+    end
+    
+    # Parses a list of consequent files of the same format
+    def parse_files(files, options = {}, &block)
+      files.each { |file| parse_file(file, options, &block) }
+    end
+    
+    # Parses a file. 
+    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
+    def parse_file(file, options = {}, &block)
+      @progress_handler.call(:started, file) if @progress_handler
+      File.open(file, 'r') { |f| parse_io(f, options, &block) }
+      @progress_handler.call(:finished, file) if @progress_handler
+    end
+    
+    def parse_stream(stream, options = {}, &block)
+      parse_io(stream, options, &block)
+    end
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def parse_io(io, options = {}, &block)
+
+      # parse every line type by default
+      line_types = options[:line_types] || file_format.line_definitions.keys
+
+      # check whether all provided line types are valid
+      unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
+      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
+      
+      puts "Parsing mode: " + (options[:combined_requests] ? 'combined requests' : 'single lines') if options[:debug]
+      
+      @current_io = io
+      @current_io.each_line do |line|
+        
+        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
+        
+        request_data = nil
+        line_types.each do |line_type|
+          line_type_definition = file_format.line_definitions[line_type]
+          break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
+        end
+
+        if request_data
+          @parsed_lines += 1
+          if @options[:combined_requests]
+            update_current_request(request_data, &block)
+          else
+            handle_request(RequestLogAnalyzer::Request.create(@file_format, request_data), &block)
+          end       
+        end
+      end
+      
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+  
+      @current_io = nil
+    end
+    
+    # Add a block to this method to install a progress handler while parsing
+    def progress=(proc)
+      @progress_handler = proc
+    end
+    
+    # Add a block to this method to install a warning handler while parsing
+    def warning=(proc)
+      @warning_handler = proc
+    end
+    
+    # This method is called by the parser if it encounteres any problems.
+    # It will call the warning handler. The default controller will pass all warnings to every
+    # aggregator that is registered and running
+    def warn(type, message)
+      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
+    end
+    
+    protected
+    
+    # Combines the different lines of a request into a single Request object.
+    # This function is only called in combined requests mode. It will start a new request when
+    # a header line is encountered en will emit the request when a footer line is encountered.
+    #
+    # - Every line that is parsed before a header line is ignored as it cannot be included in 
+    #   any request. It will emit a :no_current_request warning.
+    # - A header line that is parsed before a request is closed by a footer line, is a sign of
+    #   an unprpertly ordered file. All data that is gathered for the request until then is 
+    #   discarded, the next request is ignored as well and a :unclosed_request warning is
+    #   emitted.
+    def update_current_request(request_data, &block)
+      if header_line?(request_data)
+        unless @current_request.nil?
+          if options[:assume_correct_order]
+            handle_request(@current_request, &block)            
+            @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
+          else
+            warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
+            @current_request = nil # remove all data that was parsed, skip next request as well.
+          end
+        else
+          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
+        end
+      else
+        unless @current_request.nil?
+          @current_request << request_data
+          if footer_line?(request_data)
+            handle_request(@current_request, &block)
+            @current_request = nil 
+          end
+        else
+          warn(:no_current_request, "Parsebale line found outside of a request!")
+        end
+      end
+    end
+    
+    # Handles the parsed request by calling the request handler.
+    # The default controller will send the request to every running aggegator.
+    def handle_request(request, &block)
+      @parsed_requests += 1
+      accepted = block_given? ? yield(request) : true
+      @skipped_requests += 1 if !accepted
+    end
+    
+    # Checks whether a given line hash is a header line.
+    def header_line?(hash)
+      file_format.line_definitions[hash[:line_type]].header
+    end
+
+    # Checks whether a given line hash is a footer line.    
+    def footer_line?(hash)
+      file_format.line_definitions[hash[:line_type]].footer  
+    end 
+  end
+end

data/lib/request_log_analyzer/log_processor.rb

@@ -0,0 +1,121 @@
+module RequestLogAnalyzer
+  
+  # The Logprocessor class is used to perform simple processing actions over log files.
+  # It will go over the log file/stream line by line, pass the line to a processor and 
+  # write the result back to the output file or stream. The processor can alter the 
+  # contents of the line, remain it intact or remove it altogether, based on the current
+  # file format
+  #
+  # Currently, two processors are supported, :strip and :anonymize. 
+  #  * :strip will remove all irrelevent lines (according to the file format) from the 
+  #    sources. A compact, information packed log will remain/.
+  #  * :anonymize will anonymize sensitive information from the lines according to the 
+  #    anonymization rules in the file format. The result can be passed to third parties
+  #    without privacy concerns.
+  #
+  class LogProcessor
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    attr_reader :mode, :options, :sources
+    attr_accessor :output_file
+   
+    # Builds a logprocessor instance from the arguments given on the command line
+    # <tt>command</tt> The command hat was used to start the log processor. This can either be 
+    #   :strip or :anonymize. This will set the processing mode.
+    # <tt>arguments</tt> The parsed command line arguments (a CommandLine::Arguments instance)
+    def self.build(command, arguments)
+      
+      options = { 
+          :discard_teaser_lines => arguments[:discard_teaser_lines], 
+          :keep_junk_lines      => arguments[:keep_junk_lines], 
+        }
+          
+      log_processor = RequestLogAnalyzer::LogProcessor.new(arguments[:format].to_sym, command, options)
+      log_processor.output_file = arguments[:output] if arguments[:output]
+
+      arguments.parameters.each do |input|
+        log_processor.sources << input
+      end
+      
+      return log_processor
+    end
+    
+    # Initializes a new LogProcessor instance.
+    # <tt>format</tt> The file format to use (e.g. :rails).
+    # <tt>mode</tt> The processing mode (:anonymize or :strip)
+    # <tt>options</tt> A hash with options to take into account
+    def initialize(format, mode, options = {})
+      @options = options
+      @mode    = mode
+      @sources = []
+      $output_file = nil
+      self.register_file_format(format)
+    end
+    
+    # Processes input files by opening it and sending the filestream to <code>process_io</code>,
+    # in which the actual processing is performed.
+    # <tt>file</tt> The file to process
+    def process_file(file)
+      File.open(file, 'r') { |file| process_io(file) }
+    end
+
+    # Processes an input stream by iteration over each line and processing it according to
+    # the current operation mode (:strip, :anonymize)
+    # <tt>io</tt> The IO instance to process.
+    def process_io(io)
+      case mode
+        when :strip;     io.each_line { |line| @output << strip_line(line) }
+        when :anonymize; io.each_line { |line| @output << anonymize_line(line) }
+      end
+    end
+    
+    # Returns the line itself if the string matches any of the line definitions. If no match is
+    # found, an empty line is returned, which will strip the line from the output.
+    # <tt>line</tt> The line to strip
+    def strip_line(line)
+      file_format.line_definitions.any? { |name, definition| definition =~ line } ? line : ""
+    end
+
+    # Returns an anonymized version of the provided line. This can be a copy of the line it self, 
+    # an empty string or a string in which some substrings are substituted for anonymized values.
+    # <tt>line</tt> The line to anonymize
+    def anonymize_line(line)
+      anonymized_line = nil
+      file_format.line_definitions.detect { |name, definition| anonymized_line = definition.anonymize(line, options) }
+      
+      if anonymized_line
+        return anonymized_line
+      elsif options[:keep_junk_lines]
+        return line
+      else
+        return ""
+      end
+    end
+
+    # Runs the log processing by setting up the output stream and iterating over all the
+    # input sources. Input sources can either be filenames (String instances) or IO streams 
+    # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable.
+    def run!
+      if @output_file.nil?
+        @output = $stdout 
+      else
+        @output = File.new(@output_file, 'a')        
+      end
+      
+      @sources.each do |source|
+        if source.kind_of?(String) && File.exist?(source)
+          process_file(source)
+        elsif source.kind_of?(IO)
+          process_io(source)
+        elsif ['-', 'STDIN'].include?(source)
+          process_io($stdin)        
+        end
+      end
+    
+    ensure
+      @output.close if @output.kind_of?(File)
+    end
+  end
+
+end

data/lib/request_log_analyzer/request.rb

@@ -0,0 +1,115 @@
+module RequestLogAnalyzer
+  
+  # The Request class represents a parsed request from the log file. 
+  # Instances are created by the LogParser and are passed to the different aggregators, so they
+  # can do their aggregating work. 
+  #
+  # Note that RequestLogAnalyzer can run in two modes:
+  # - Single line mode: every parsed line is regarded as a request. Request::single_line? will 
+  #   return true in this case
+  # - Combined requests mode: lines that belong together are grouped into one request.
+  #   Request#combined? will return true in this case.
+  #
+  # This class provides several methods to access the data that was parsed from the log files.
+  # Request#first(field_name) returns the first (only) value corresponding to the given field
+  # Request#every(field_name) returns all values corresponding to the given field name as array.
+  class Request
+  
+    include RequestLogAnalyzer::FileFormat::Awareness
+  
+    attr_reader :lines
+    attr_reader :attributes
+        
+    # Initializes a new Request object. 
+    # It will apply the the provided FileFormat module to this instance.
+    def initialize(file_format)
+      @lines = []
+      @attributes = {}
+      register_file_format(file_format)
+    end
+    
+    # Creates a new request that was parsed from the log with the given FileFormat. The hashes
+    # that are passed to this function are added as lines to this request.
+    def self.create(file_format, *hashes)
+      request = self.new(file_format)
+      hashes.flatten.each { |hash| request << hash }
+      return request
+    end
+     
+    # Adds another line to the request.
+    # The line should be provides as a hash of the fields parsed from the line.
+    def << (request_info_hash)
+      @lines << request_info_hash
+      @attributes = request_info_hash.merge(@attributes)
+    end
+    
+    # Checks whether the given line type was parsed from the log file for this request
+    def has_line_type?(line_type)
+      return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
+      
+      @lines.detect { |l| l[:line_type] == line_type.to_sym }
+    end
+    
+    alias :=~ :has_line_type?
+    
+    # Returns the value that was captured for the "field" of this request.
+    # This function will return the first value that was captured if the field
+    # was captured in multiple lines for a combined request.
+    def first(field)
+      @attributes[field]
+    end
+    
+    alias :[] :first
+    
+    # Returns an array of all the "field" values that were captured for this request
+    def every(field)
+      @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
+    end    
+    
+    # Returns true if this request does not yet contain any parsed lines. This should only occur 
+    # during parsing. An empty request should never be sent to the aggregators
+    def empty?
+      @lines.length == 0
+    end
+    
+    # Checks whether this request contains exactly one line. This means that RequestLogAnalyzer
+    # is running in single_line mode.
+    def single_line?
+      @lines.length == 1
+    end
+    
+    # Checks whether this request contains more than one line. This means that RequestLogAnalyzer
+    # is runring in combined requests mode.
+    def combined?
+      @lines.length > 1
+    end
+    
+    # Checks whether this request is completed. A completed request contains both a parsed header
+    # line and a parsed footer line. Not that calling this function in single line mode will always 
+    # return false.
+    def completed?
+      puts attributes[:method]
+
+      header_found, footer_found = false, false
+      @lines.each do |line| 
+        line_def = file_format.line_definitions[line[:line_type]]
+        header_found = true if line_def.header
+        footer_found = true if line_def.footer        
+      end
+      header_found && footer_found
+      
+    end
+    
+    # Returns the line type of the parsed line of this request.
+    # This function can only be called in single line mode.
+    def line_type
+      raise "Not a single line request!" unless single_line?
+      lines.first[:line_type]
+    end
+    
+    # Returns the first timestamp encountered in a request. 
+    def timestamp
+      first(:timestamp)
+    end
+  end
+end

data/lib/request_log_analyzer/source/base.rb

@@ -0,0 +1,42 @@
+module RequestLogAnalyzer::Source
+  class Base
+    
+    include RequestLogAnalyzer::FileFormat::Awareness
+
+    # A hash of options
+    attr_reader :options
+
+    # The current Request object that is being parsed
+    attr_reader :current_request
+
+    # The total number of parsed lines
+    attr_reader :parsed_lines
+
+    # The total number of parsed requests.
+    attr_reader :parsed_requests
+
+    # The number of skipped requests because of date constraints
+    attr_reader :skipped_requests
+
+    # Base source class used to filter input requests.
+
+    # Initializer
+    # <tt>format</tt> The file format
+    # <tt>options</tt> Are passed to the filters.
+    def initialize(format, options = {})
+      @options    = options
+      register_file_format(format)
+    end
+    
+    def prepare
+    end
+    
+    def requests(&block)
+      return true
+    end
+
+    def finalize
+    end
+
+  end
+end