ngmoco-request-log-analyzer 1.4.2

data/lib/request_log_analyzer/request.rb

@@ -0,0 +1,175 @@
+module RequestLogAnalyzer
+
+  # The Request class represents a parsed request from the log file.
+  # Instances are created by the LogParser and are passed to the different aggregators, so they
+  # can do their aggregating work.
+  #
+  # This class provides several methods to access the data that was parsed from the log files.
+  # Request#first(field_name) returns the first (only) value corresponding to the given field
+  # Request#every(field_name) returns all values corresponding to the given field name as array.
+  class Request
+
+    module Converters
+
+      # Default converter function, which converts the parsed strings to a native Ruby type
+      # using the type indication in the line definition. It will use a custom connverter
+      # method if one is available.
+      def convert_value(value, capture_definition)
+        return capture_definition[:default] if value.nil?
+        custom_converter_method = :"convert_#{capture_definition[:type]}"
+        send(custom_converter_method, value, capture_definition)
+      end
+
+      def convert_string(value, capture_definition);  value; end
+      def convert_float(value, capture_definition);   value.to_f; end
+      def convert_decimal(value, capture_definition); value.to_f; end
+      def convert_int(value, capture_definition);     value.to_i; end
+      def convert_integer(value, capture_definition); value.to_i; end
+      def convert_sym(value, capture_definition);     value.to_sym; end
+      def convert_symbol(value, capture_definition);  value.to_sym; end
+
+      # Converts :eval field, which should evaluate to a hash.
+      def convert_eval(value, capture_definition)
+        eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
+      rescue SyntaxError
+        nil
+      end
+
+      # Slow default method to parse timestamps.
+      # Reimplement this function in a file format specific Request class
+      # to improve the timestamp parsing speed.
+      def convert_timestamp(value, capture_definition)
+        DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i
+      end
+
+      # Converts traffic fields to (whole) bytes based on the given unit.
+      def convert_traffic(value, capture_definition)
+        case capture_definition[:unit]
+        when nil, :b, :B, :byte      then value.to_i
+        when :GB, :G, :gigabyte      then (value.to_f * 1000_000_000).round
+        when :GiB, :gibibyte         then (value.to_f * (2 ** 30)).round
+        when :MB, :M, :megabyte      then (value.to_f * 1000_000).round
+        when :MiB, :mebibyte         then (value.to_f * (2 ** 20)).round
+        when :KB, :K, :kilobyte, :kB then (value.to_f * 1000).round
+        when :KiB, :kibibyte         then (value.to_f * (2 ** 10)).round
+        else raise "Unknown traffic unit"
+        end
+      end
+
+      # Convert duration fields to float, and make sure the values are in seconds.
+      def convert_duration(value, capture_definition)
+        case capture_definition[:unit]
+        when nil, :sec, :s     then value.to_f
+        when :microsec, :musec then value.to_f / 1000000.0
+        when :msec, :millisec  then value.to_f / 1000.0
+        else raise "Unknown duration unit"
+        end
+      end
+    end
+
+    # Install the default converter methods
+    include Converters
+
+    attr_reader :lines, :attributes, :file_format
+
+    # Initializes a new Request object.
+    # It will apply the the provided FileFormat module to this instance.
+    def initialize(file_format, attributes = {})
+      @lines       = []
+      @attributes  = attributes
+      @file_format = file_format
+    end
+
+    # Creates a new request that was parsed from the log with the given FileFormat. The hashes
+    # that are passed to this function are added as lines to this request.
+    def self.create(file_format, *hashes)
+      request = self.new(file_format)
+      hashes.flatten.each { |hash| request << hash }
+      return request
+    end
+
+    # Adds another line to the request when it is parsed in the LogParser.
+    #
+    # The line should be provided as a hash with the attributes line_definition, :captures,
+    # :lineno and :source set. This function is called from LogParser.
+    def add_parsed_line (parsed_line)
+      value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
+      value_hash[:line_type] = parsed_line[:line_definition].name
+      value_hash[:lineno] = parsed_line[:lineno]
+      value_hash[:source] = parsed_line[:source]
+      add_line_hash(value_hash)
+    end
+
+    # Adds another line to the request using a plain hash.
+    #
+    # The line should be provides as a hash of the fields parsed from the line.
+    def add_line_hash(value_hash)
+      @lines << value_hash
+      @attributes = value_hash.merge(@attributes)
+    end
+
+    # Adds another line to the request. This method switches automatically between
+    # the add_line_hash and add_parsed_line based on the keys of the provided hash.
+    def <<(hash)
+      hash[:line_definition] ? add_parsed_line(hash) : add_line_hash(hash)
+    end
+
+    # Checks whether the given line type was parsed from the log file for this request
+    def has_line_type?(line_type)
+      return true if @lines.length == 1 && @lines[0][:line_type] == line_type.to_sym
+      @lines.detect { |l| l[:line_type] == line_type.to_sym }
+    end
+
+    alias :=~ :has_line_type?
+
+    # Returns the value that was captured for the "field" of this request.
+    # This function will return the first value that was captured if the field
+    # was captured in multiple lines
+    def first(field)
+      @attributes[field]
+    end
+
+    alias :[] :first
+
+    # Returns an array of all the "field" values that were captured for this request
+    def every(field)
+      @lines.inject([]) { |result, fields| result << fields[field] if fields.has_key?(field); result }
+    end
+
+    # Returns true if this request does not yet contain any parsed lines. This should only occur
+    # during parsing. An empty request should never be sent to the aggregators
+    def empty?
+      @lines.length == 0
+    end
+
+    # Checks whether this request is completed. A completed request contains both a parsed header
+    # line and a parsed footer line. Not that calling this function in single line mode will always
+    # return false.
+    def completed?
+      header_found, footer_found = false, false
+      @lines.each do |line|
+        line_def = file_format.line_definitions[line[:line_type]]
+        header_found = true if line_def.header
+        footer_found = true if line_def.footer
+      end
+      header_found && footer_found
+    end
+
+    # This function is called before a Requests is yielded.
+    def validate
+    end
+
+    # Returns the first timestamp encountered in a request.
+    def timestamp
+      first(:timestamp)
+    end
+
+    def first_lineno
+      @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.min
+    end
+
+    def last_lineno
+      @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.max
+    end
+  end
+end

data/lib/request_log_analyzer/source.rb

@@ -0,0 +1,72 @@
+# The RequestLogAnalyzer::Source module contains all functionality that loads requests from a given source
+# and feed them to the pipeline for further processing. The requests (see RequestLogAnalyzer::Request) that
+# will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then
+# fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of
+# the RequestLogAnalyzer chain.
+#
+# - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class.
+# - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
+module RequestLogAnalyzer::Source
+
+  # Loads constants that reside in the RequestLogAnalyzer::Source namespace. This function uses
+  # RequestLogAnalyzer::load_default_class_file to load the file in which the constant is declared.
+  # <tt>const</tt>:: The constant to load in the RequestLogAnalyzer::Source namespace.
+  def self.const_missing(const)
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end
+
+  # The base Source class. All other sources should inherit from this class.
+  #
+  # A source implememtation should at least implement the each_request method, which should yield
+  # RequestLogAnalyzer::Request instances that will be fed through the pipleine.
+  class Base
+
+    # A hash of options
+    attr_reader :options
+
+    # The current Request object that is being parsed
+    attr_reader :current_request
+
+    # The total number of parsed lines
+    attr_reader :parsed_lines
+
+    # The number of skipped lines because of warnings
+    attr_reader :skipped_lines
+
+    # The total number of parsed requests.
+    attr_reader :parsed_requests
+
+    # The total number of skipped requests because of filters.
+    attr_reader :skipped_requests
+
+    # The FileFormat instance that describes the format of this source.
+    attr_reader :file_format
+
+    # Initializer, which will register the file format and save any options given as a hash.
+    # <tt>format</tt>:: The file format instance
+    # <tt>options</tt>:: A hash of options that can be used by a specific Source implementation
+    def initialize(format, options = {})
+      @options     = options
+      @file_format = format
+    end
+
+    # The prepare method is called before the RequestLogAnalyzer::Source::Base#each_request method is called.
+    # Use this method to implement any initialization that should occur before this source can produce Request
+    # instances.
+    def prepare
+    end
+
+    # This function is called to actually produce the requests that will be send into the pipeline.
+    # The implementation should yield instances of RequestLogAnalyzer::Request.
+    # <tt>options</tt>:: A Hash of options that can be used in the implementation.
+    def each_request(options = {}, &block) # :yields: request
+      return true
+    end
+
+    # This function is called after RequestLogAnalyzer::Source::Base#each_request finished. Any code to
+    # wrap up, free resources, etc. can be put in this method.
+    def finalize
+    end
+
+  end
+end

data/lib/request_log_analyzer/source/database_loader.rb

@@ -0,0 +1,87 @@
+require 'rubygems'
+require 'activerecord'
+
+module RequestLogAnalyzer::Source
+
+  # Active Resource hook
+  class Request < ActiveRecord::Base
+    has_many :completed_lines
+    has_many :processing_lines
+    def convert(file_format)
+      send_attributes = self.attributes
+      send_attributes.merge!(self.completed_lines.first.attributes) if self.completed_lines.first
+      send_attributes.merge!(self.processing_lines.first.attributes) if self.processing_lines.first
+      return RequestLogAnalyzer::Request.new(file_format, send_attributes)
+    end
+  end
+
+  class CompletedLine < ActiveRecord::Base
+    belongs_to :request
+  end
+
+  class ProcessingLine < ActiveRecord::Base
+    belongs_to :request
+  end
+
+  # The Database class gets log data from the database.
+  class DatabaseLoader < Base
+
+    attr_reader :source_files, :file_format, :requests
+
+    # Initializes the log file parser instance.
+    # It will apply the language specific FileFormat module to this instance. It will use the line
+    # definitions in this module to parse any input that it is given (see parse_io).
+    #
+    # <tt>format</tt>:: The current file format instance
+    # <tt>options</tt>:: A hash of options that are used by the parser
+    def initialize(format, options = {})
+      super(format, options)
+      @source_files     = options[:source_files]
+      @parsed_requests  = 0
+      @requests         = []
+    end
+
+    # Reads the input, which can either be a file, sequence of files or STDIN to parse
+    # lines specified in the FileFormat. This lines will be combined into Request instances,
+    # that will be yielded. The actual parsing occurs in the parse_io method.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def each_request(options = {}, &block) # :yields: request
+      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => @source_files)
+
+      @progress_handler.call(:started, @source_files) if @progress_handler
+        RequestLogAnalyzer::Source::Request.find(:all).each do |request|
+          @parsed_requests += 1
+          @progress_handler.call(:progress, @parsed_requests) if @progress_handler
+
+          yield request.convert(self.file_format)
+        end
+
+      @progress_handler.call(:finished, @source_files) if @progress_handler
+    end
+
+    # Add a block to this method to install a progress handler while parsing.
+    # <tt>proc</tt>:: The proc that will be called to handle progress update messages
+    def progress=(proc)
+      @progress_handler = proc
+    end
+
+    # Add a block to this method to install a warning handler while parsing,
+    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages
+    def warning=(proc)
+      @warning_handler = proc
+    end
+
+    # This method is called by the parser if it encounteres any parsing problems.
+    # It will call the installed warning handler if any.
+    #
+    # By default, RequestLogAnalyzer::Controller will install a warning handler
+    # that will pass the warnings to each aggregator so they can do something useful
+    # with it.
+    #
+    # <tt>type</tt>:: The warning type (a Symbol)
+    # <tt>message</tt>:: A message explaining the warning
+    def warn(type, message)
+      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
+    end
+  end
+end

data/lib/request_log_analyzer/source/log_parser.rb

@@ -0,0 +1,274 @@
+module RequestLogAnalyzer::Source
+
+  # The LogParser class reads log data from a given source and uses a file format definition
+  # to parse all relevent information about requests from the file. A FileFormat module should
+  # be provided that contains the definitions of the lines that occur in the log data.
+  #
+  # De order in which lines occur is used to combine lines to a single request. If these lines
+  # are mixed, requests cannot be combined properly. This can be the case if data is written to
+  # the log file simultaneously by different mongrel processes. This problem is detected by the
+  # parser. It will emit warnings when this occurs. LogParser supports multiple parse strategies
+  # that deal differently with this problem.
+  class LogParser < Base
+
+    include Enumerable
+
+    # The default parse strategy that will be used to parse the input.
+    DEFAULT_PARSE_STRATEGY = 'assume-correct'
+
+    # All available parse strategies.
+    PARSE_STRATEGIES = ['cautious', 'assume-correct']
+
+    attr_reader :source_files, :current_file, :current_lineno
+
+    # Initializes the log file parser instance.
+    # It will apply the language specific FileFormat module to this instance. It will use the line
+    # definitions in this module to parse any input that it is given (see parse_io).
+    #
+    # <tt>format</tt>:: The current file format instance
+    # <tt>options</tt>:: A hash of options that are used by the parser
+    def initialize(format, options = {})
+      super(format, options)
+      @parsed_lines     = 0
+      @parsed_requests  = 0
+      @skipped_lines    = 0
+      @skipped_requests = 0
+      @current_request  = nil
+      @current_source   = nil
+      @current_file     = nil
+      @current_lineno   = nil
+      @source_files     = options[:source_files]
+      @progress_handler = nil
+
+      @options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
+      raise "Unknown parse strategy" unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
+    end
+
+    # Reads the input, which can either be a file, sequence of files or STDIN to parse
+    # lines specified in the FileFormat. This lines will be combined into Request instances,
+    # that will be yielded. The actual parsing occurs in the parse_io method.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def each_request(options = {}, &block) # :yields: :request, request
+
+      case @source_files
+      when IO
+        if @source_files == $stdin
+          puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
+        end
+        parse_stream(@source_files, options, &block)
+      when String
+        parse_file(@source_files, options, &block)
+      when Array
+        parse_files(@source_files, options, &block)
+      else
+        raise "Unknown source provided"
+      end
+    end
+
+    # Make sure the Enumerable methods work as expected
+    alias_method :each, :each_request
+
+    # Parses a list of subsequent files of the same format, by calling parse_file for every
+    # file in the array.
+    # <tt>files</tt>:: The Array of files that should be parsed
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def parse_files(files, options = {}, &block) # :yields: request
+      files.each { |file| parse_file(file, options, &block) }
+    end
+
+    # Check if a file has a compressed extention in the filename.
+    # If recognized, return the command string used to decompress the file
+    def decompress_file?(filename)
+      nice_command = "nice -n 5"
+
+      return "#{nice_command} gunzip -c -d #{filename}" if filename.match(/\.tar.gz$/) || filename.match(/\.tgz$/) || filename.match(/\.gz$/)
+      return "#{nice_command} bunzip2 -c -d #{filename}" if filename.match(/\.bz2$/)
+      return "#{nice_command} unzip -p #{filename}" if filename.match(/\.zip$/)
+
+      return ""
+    end
+
+    # Parses a log file. Creates an IO stream for the provided file, and sends it to parse_io for
+    # further handling. This method supports progress updates that can be used to display a progressbar
+    #
+    # If the logfile is compressed, it is uncompressed to stdout and read.
+    # TODO: Check if IO.popen encounters problems with the given command line.
+    # TODO: Fix progress bar that is broken for IO.popen, as it returns a single string.
+    #
+    # <tt>file</tt>:: The file that should be parsed.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def parse_file(file, options = {}, &block)
+
+      @current_source = File.expand_path(file)
+      @source_changes_handler.call(:started, @current_source) if @source_changes_handler
+
+      if decompress_file?(file).empty?
+
+        @progress_handler = @dormant_progress_handler
+        @progress_handler.call(:started, file) if @progress_handler
+
+        File.open(file, 'r') { |f| parse_io(f, options, &block) }
+
+        @progress_handler.call(:finished, file) if @progress_handler
+        @progress_handler = nil
+      else
+        IO.popen(decompress_file?(file), 'r') { |f| parse_io(f, options, &block) }
+      end
+
+      @source_changes_handler.call(:finished, @current_source) if @source_changes_handler
+
+      @current_source = nil
+
+    end
+
+    # Parses an IO stream. It will simply call parse_io. This function does not support progress updates
+    # because the length of a stream is not known.
+    # <tt>stream</tt>:: The IO stream that should be parsed.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def parse_stream(stream, options = {}, &block)
+      parse_io(stream, options, &block)
+    end
+
+    # This method loops over each line of the input stream. It will try to parse this line as any of
+    # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
+    # It will then combine these parsed line into requests using heuristics. These requests (see
+    # RequestLogAnalyzer::Request) will then be yielded for further processing in the pipeline.
+    #
+    # - RequestLogAnalyzer::LineDefinition#matches is called to test if a line matches a line definition of the file format.
+    # - update_current_request is used to combine parsed lines into requests using heuristics.
+    # - The method will yield progress updates if a progress handler is installed using progress=
+    # - The method will yield parse warnings if a warning handler is installed using warning=
+    #
+    # <tt>io</tt>:: The IO instance to use as source
+    # <tt>options</tt>:: A hash of options that can be used by the parser.
+    def parse_io(io, options = {}, &block) # :yields: request
+      @current_lineno = 1
+      while line = io.gets
+        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
+
+        if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
+          @parsed_lines += 1
+          update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
+        end
+
+        @current_lineno += 1
+      end
+
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+      @current_lineno = nil
+    end
+
+    # Add a block to this method to install a progress handler while parsing.
+    # <tt>proc</tt>:: The proc that will be called to handle progress update messages
+    def progress=(proc)
+      @dormant_progress_handler = proc
+    end
+
+    # Add a block to this method to install a warning handler while parsing,
+    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages
+    def warning=(proc)
+      @warning_handler = proc
+    end
+
+    # Add a block to this method to install a source change handler while parsing,
+    # <tt>proc</tt>:: The proc that will be called to handle source changes
+    def source_changes=(proc)
+      @source_changes_handler = proc
+    end
+
+    # This method is called by the parser if it encounteres any parsing problems.
+    # It will call the installed warning handler if any.
+    #
+    # By default, RequestLogAnalyzer::Controller will install a warning handler
+    # that will pass the warnings to each aggregator so they can do something useful
+    # with it.
+    #
+    # <tt>type</tt>:: The warning type (a Symbol)
+    # <tt>message</tt>:: A message explaining the warning
+    def warn(type, message)
+      @warning_handler.call(type, message, @current_lineno) if @warning_handler
+    end
+
+    protected
+
+    # Combines the different lines of a request into a single Request object. It will start a
+    # new request when a header line is encountered en will emit the request when a footer line
+    # is encountered.
+    #
+    # Combining the lines is done using heuristics. Problems can occur in this process. The
+    # current parse strategy defines how these cases are handled.
+    #
+    # When using the 'assume-correct' parse strategy (default):
+    # - Every line that is parsed before a header line is ignored as it cannot be included in
+    #   any request. It will emit a :no_current_request warning.
+    # - If a header line is found before the previous requests was closed, the previous request
+    #   will be yielded and a new request will be started.
+    #
+    # When using the 'cautious' parse strategy:
+    # - Every line that is parsed before a header line is ignored as it cannot be included in
+    #   any request. It will emit a :no_current_request warning.
+    # - A header line that is parsed before a request is closed by a footer line, is a sign of
+    #   an unproperly ordered file. All data that is gathered for the request until then is
+    #   discarded and the next request is ignored as well. An :unclosed_request warning is
+    #   emitted.
+    #
+    # <tt>request_data</tt>:: A hash of data that was parsed from the last line.
+    def update_current_request(request_data, &block) # :yields: request
+      if header_line?(request_data)
+        if @current_request
+          case options[:parse_strategy]
+          when 'assume-correct'
+            handle_request(@current_request, &block)
+            @current_request = @file_format.request(request_data)
+          when 'cautious'
+            @skipped_lines += 1
+            warn(:unclosed_request, "Encountered header line (#{request_data[:line_definition].name.inspect}), but previous request was not closed!")
+            @current_request = nil # remove all data that was parsed, skip next request as well.
+          end
+        elsif footer_line?(request_data)
+          handle_request(@file_format.request(request_data), &block)
+        else
+          @current_request = @file_format.request(request_data)
+        end
+      else
+        if @current_request
+          @current_request << request_data
+          if footer_line?(request_data)
+            handle_request(@current_request, &block) # yield @current_request
+            @current_request = nil
+          end
+        else
+          @skipped_lines += 1
+          warn(:no_current_request, "Parsebale line (#{request_data[:line_definition].name.inspect}) found outside of a request!")
+        end
+      end
+    end
+
+    # Handles the parsed request by sending it into the pipeline.
+    #
+    # - It will call RequestLogAnalyzer::Request#validate on the request instance
+    # - It will send the request into the pipeline, checking whether it was accepted by all the filters.
+    # - It will update the parsed_requests and skipped_requests variables accordingly
+    #
+    # <tt>request</tt>:: The parsed request instance (RequestLogAnalyzer::Request)
+    def handle_request(request, &block) # :yields: :request, request
+      @parsed_requests += 1
+      request.validate
+      accepted = block_given? ? yield(request) : true
+      @skipped_requests += 1 unless accepted
+    end
+
+    # Checks whether a given line hash is a header line according to the current file format.
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
+    def header_line?(hash)
+      hash[:line_definition].header
+    end
+
+    # Checks whether a given line hash is a footer line  according to the current file format.
+    # <tt>hash</tt>:: A hash of data that was parsed from the line.
+    def footer_line?(hash)
+      hash[:line_definition].footer
+    end
+  end
+
+end