ngmoco-request-log-analyzer 1.4.2

data/lib/request_log_analyzer/file_format/rails_development.rb

@@ -0,0 +1,12 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The RailsDevelopment FileFormat is an extention to the default Rails file format. It includes
+  # all lines of the normal Rails file format, but parses SQL queries and partial rendering lines
+  # as well.
+  class RailsDevelopment < Rails
+    def self.create
+      puts 'DEPRECATED: use --rails-format development instead!'
+      super('development')
+    end
+  end
+end

data/lib/request_log_analyzer/filter.rb

@@ -0,0 +1,30 @@
+module RequestLogAnalyzer::Filter
+
+  # Filter class loader using const_missing
+  # This function will automatically load the class file based on the name of the class
+  def self.const_missing(const)
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end
+
+  # Base filter class used to filter input requests.
+  # All filters should interit from this base.
+  class Base
+
+    attr_reader :file_format, :options
+
+    # Initializer
+    # <tt>format</tt> The file format
+    # <tt>options</tt> Are passed to the filters.
+    def initialize(format, options = {})
+      @file_format = format
+      @options     = options
+    end
+
+    # Return the request if the request should be kept.
+    # Return nil otherwise.
+    def filter(request)
+      request
+    end
+  end
+
+end

data/lib/request_log_analyzer/filter/anonymize.rb

@@ -0,0 +1,39 @@
+module RequestLogAnalyzer::Filter
+
+  # Filter to anonymize parsed values
+  # Options
+  # * <tt>:mode</tt> :reject or :accept.
+  # * <tt>:field</tt> Specific field to accept or reject.
+  # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
+  class Anonymize < Base
+
+    def generate_random_ip
+      "#{rand(256)}.#{rand(256)}.#{rand(256)}.#{rand(256)}"
+    end
+
+    def anonymize_url(value)
+      return value.sub(/^https?\:\/\/[A-Za-z0-9\.-]+\//, "http://example.com/")
+    end
+
+    def fuzz(value)
+      value * ((75 + rand(50)) / 100.0)
+    end
+
+    def filter(request)
+      # TODO: request.attributes is bad practice
+      request.attributes.each do |key, value|
+        if key == :ip
+          request.attributes[key] = generate_random_ip
+        elsif key == :url
+          request.attributes[key] = anonymize_url(value)
+        elsif [ :duration, :view, :db, :type, :after_filters_time, :before_filters_time,
+                :action_time].include?(key)
+          request.attributes[key] = fuzz(value)
+        end
+      end
+
+      return request
+    end
+  end
+
+end

data/lib/request_log_analyzer/filter/field.rb

@@ -0,0 +1,42 @@
+module RequestLogAnalyzer::Filter
+
+  # Filter to select or reject a specific field
+  # Options
+  # * <tt>:mode</tt> :reject or :accept.
+  # * <tt>:field</tt> Specific field to accept or reject.
+  # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
+  class Field < Base
+
+    attr_reader :field, :value, :mode
+
+    def initialize(file_format, options = {})
+      super(file_format, options)
+      setup_filter
+    end
+
+    # Setup mode, field and value.
+    def setup_filter
+      @mode = (@options[:mode] || :accept).to_sym
+      @field = @options[:field].to_sym
+
+      # Convert the timestamp to the correct formats for quick timestamp comparisons
+      if @options[:value].kind_of?(String) && @options[:value][0, 1] == '/' && @options[:value][-1, 1] == '/'
+        @value = Regexp.new(@options[:value][1..-2])
+      else
+        @value = @options[:value] # TODO: convert value?
+      end
+    end
+
+    # Keep request if @mode == :select and request has the field and value.
+    # Drop request if @mode == :reject and request has the field and value.
+    # Returns nil otherwise.
+    # <tt>request</tt> Request Object
+    def filter(request)
+      found_field = request.every(@field).any? { |value| @value === value.to_s }
+      return nil if !found_field && @mode == :select
+      return nil if found_field && @mode == :reject
+      return request
+    end
+  end
+
+end

data/lib/request_log_analyzer/filter/timespan.rb

@@ -0,0 +1,45 @@
+module RequestLogAnalyzer::Filter
+
+  # Reject all requests not in given timespan
+  # Options
+  # * <tt>:after</tt> Only keep requests after this DateTime.
+  # * <tt>:before</tt> Only keep requests before this DateTime.
+  class Timespan < Base
+
+    attr_reader :before, :after
+
+    def initialize(file_format, options = {})
+      @after  = nil
+      @before = nil
+      super(file_format, options)
+      setup_filter
+    end
+
+
+    # Convert the timestamp to the correct formats for quick timestamp comparisons.
+    # These are stored in the before and after attr_reader fields.
+    def setup_filter
+      @after  = @options[:after].strftime('%Y%m%d%H%M%S').to_i  if options[:after]
+      @before = @options[:before].strftime('%Y%m%d%H%M%S').to_i if options[:before]
+    end
+
+    # Returns request if:
+    #   * @after <= request.timestamp <= @before
+    #   * @after <= request.timestamp
+    #   * request.timestamp <= @before
+    # Returns nil otherwise
+    # <tt>request</tt> Request object.
+    def filter(request)
+      if @after && @before && request.timestamp <= @before && @after <= request.timestamp
+        return request
+      elsif @after && @before.nil? && @after <= request.timestamp
+        return request
+      elsif @before && @after.nil? && request.timestamp <= @before
+        return request
+      end
+
+      return nil
+    end
+  end
+
+end

data/lib/request_log_analyzer/line_definition.rb

@@ -0,0 +1,111 @@
+module RequestLogAnalyzer
+
+  # The line definition class is used to specify what lines should be parsed from the log file.
+  # It contains functionality to match a line against the definition and parse the information
+  # from this line. This is used by the LogParser class when parsing a log file..
+  class LineDefinition
+
+    class Definer
+
+      attr_accessor :line_definitions
+
+      def initialize
+        @line_definitions = {}
+      end
+
+      def initialize_copy(other)
+        @line_definitions = other.line_definitions.dup
+      end
+
+      def method_missing(name, *args, &block)
+        if block_given?
+          @line_definitions[name] = RequestLogAnalyzer::LineDefinition.define(name, &block)
+        else
+          @line_definitions[name] = RequestLogAnalyzer::LineDefinition.new(name, args.first)
+        end
+      end
+    end
+
+    attr_reader :name
+    attr_accessor :teaser, :regexp, :captures
+    attr_accessor :header, :footer
+
+    alias_method :header?, :header
+    alias_method :footer?, :footer
+
+    # Initializes the LineDefinition instance with a hash containing the different elements of
+    # the definition.
+    def initialize(name, definition = {})
+      @name     = name
+      @captures = []
+      @teaser   = nil
+      definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
+    end
+
+    def self.define(name, &block)
+      definition = self.new(name)
+      yield(definition) if block_given?
+      return definition
+    end
+
+    # Checks whether a given line matches this definition.
+    # It will return false if a line does not match. If the line matches, a hash is returned
+    # with all the fields parsed from that line as content.
+    # If the line definition has a teaser-check, a :teaser_check_failed warning will be emitted
+    # if this teaser-check is passed, but the full regular exprssion does not ,atch.
+    def matches(line, &warning_handler)
+      if @teaser.nil? || @teaser =~ line
+        if match_data = line.match(@regexp)
+          return { :line_definition => self, :captures => match_data.captures}
+        else
+          if @teaser && warning_handler
+            warning_handler.call(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")
+          end
+          return false
+        end
+      else
+        return false
+      end
+    end
+
+    alias :=~ :matches
+
+    # matches the line and converts the captured values using the request's
+    # convert_value function.
+    def match_for(line, request, &warning_handler)
+      if match_info = matches(line, &warning_handler)
+        convert_captured_values(match_info[:captures], request)
+      else
+        false
+      end
+    end
+
+    # Updates a captures hash using the converters specified in the request
+    # and handle the :provides option in the line definition.
+    def convert_captured_values(values, request)
+      value_hash = {}
+      captures.each_with_index do |capture, index|
+
+        # convert the value using the request convert_value function
+        converted = request.convert_value(values[index], capture)
+        value_hash[capture[:name]] ||= converted
+
+        # Add items directly to the resulting hash from the converted value
+        # if it is a hash and they are set in the :provides hash for this line definition
+        if converted.kind_of?(Hash) && capture[:provides].kind_of?(Hash)
+          capture[:provides].each do |name, type|
+            value_hash[name] ||= request.convert_value(converted[name], { :type => type })
+          end
+        end
+      end
+      return value_hash
+    end
+
+    # Returns true if this line captures values of the given name
+    def captures?(name)
+      captures.any? { |c| c[:name] == name }
+    end
+
+  end
+
+end

data/lib/request_log_analyzer/log_processor.rb

@@ -0,0 +1,99 @@
+module RequestLogAnalyzer
+
+  # The Logprocessor class is used to perform simple processing actions over log files.
+  # It will go over the log file/stream line by line, pass the line to a processor and
+  # write the result back to the output file or stream. The processor can alter the
+  # contents of the line, remain it intact or remove it altogether, based on the current
+  # file format
+  #
+  # Currently, one processors is supported:
+  #  * :strip will remove all irrelevent lines (according to the file format) from the
+  #    sources. A compact, information packed log will remain/.
+  #
+  class LogProcessor
+
+    attr_reader :mode, :options, :sources, :file_format
+    attr_accessor :output_file
+
+    # Builds a logprocessor instance from the arguments given on the command line
+    # <tt>command</tt> The command hat was used to start the log processor. This will set the
+    #    processing mode. Currently, only :strip is supported.
+    # <tt>arguments</tt> The parsed command line arguments (a CommandLine::Arguments instance)
+    def self.build(command, arguments)
+
+      options = {
+          :discard_teaser_lines => arguments[:discard_teaser_lines],
+          :keep_junk_lines      => arguments[:keep_junk_lines],
+        }
+
+      log_processor = RequestLogAnalyzer::LogProcessor.new(arguments[:format].to_sym, command, options)
+      log_processor.output_file = arguments[:output] if arguments[:output]
+
+      arguments.parameters.each do |input|
+        log_processor.sources << input
+      end
+
+      return log_processor
+    end
+
+    # Initializes a new LogProcessor instance.
+    # <tt>format</tt> The file format to use (e.g. :rails).
+    # <tt>mode</tt> The processing mode
+    # <tt>options</tt> A hash with options to take into account
+    def initialize(format, mode, options = {})
+      @options     = options
+      @mode        = mode
+      @sources     = []
+      @file_format = format
+      $output_file = nil
+    end
+
+    # Processes input files by opening it and sending the filestream to <code>process_io</code>,
+    # in which the actual processing is performed.
+    # <tt>file</tt> The file to process
+    def process_file(file)
+      File.open(file, 'r') { |file| process_io(file) }
+    end
+
+    # Processes an input stream by iteration over each line and processing it according to
+    # the current operation mode
+    # <tt>io</tt> The IO instance to process.
+    def process_io(io)
+      case mode
+        when :strip;     io.each_line { |line| @output << strip_line(line) }
+      end
+    end
+
+    # Returns the line itself if the string matches any of the line definitions. If no match is
+    # found, an empty line is returned, which will strip the line from the output.
+    # <tt>line</tt> The line to strip
+    def strip_line(line)
+      file_format.line_definitions.any? { |name, definition| definition =~ line } ? line : ""
+    end
+
+    # Runs the log processing by setting up the output stream and iterating over all the
+    # input sources. Input sources can either be filenames (String instances) or IO streams
+    # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable.
+    def run!
+      if @output_file.nil?
+        @output = $stdout
+      else
+        @output = File.new(@output_file, 'a')
+      end
+
+      @sources.each do |source|
+        if source.kind_of?(String) && File.exist?(source)
+          process_file(source)
+        elsif source.kind_of?(IO)
+          process_io(source)
+        elsif ['-', 'STDIN'].include?(source)
+          process_io($stdin)
+        end
+      end
+
+    ensure
+      @output.close if @output.kind_of?(File)
+    end
+  end
+
+end

data/lib/request_log_analyzer/mailer.rb

@@ -0,0 +1,62 @@
+module RequestLogAnalyzer
+
+  # Mail report to a specified emailaddress
+  class Mailer
+
+    attr_accessor :data, :to, :host, :content_type
+
+    # Initialize a mailer
+    # <tt>to</tt> to email address to mail to
+    # <tt>host</tt> the mailer host (defaults to localhost)
+    # <tt>options</tt> Specific style options
+    #
+    # Options
+    # * <tt>:debug</tt> Do not actually mail
+    # * <tt>:from_alias</tt> The from alias
+    # * <tt>:to_alias</tt> The to alias
+    # * <tt>:subject</tt> The message subject
+    def initialize(to, host = 'localhost', options = {})
+      require 'net/smtp'
+      @to      = to
+      @host    = host
+      @options = options
+      @data    = []
+      @content_type = nil
+    end
+
+    # Send all data in @data to the email address used during initialization.
+    # Returns array containg [message_data, from_email_address, to_email_address] of sent email.
+    def mail
+      from        = @options[:from]        || 'contact@railsdoctors.com'
+      from_alias  = @options[:from_alias]  || 'Request-log-analyzer reporter'
+      to_alias    = @options[:to_alias]    || to
+      subject     = @options[:subject]     || "Request log analyzer report - generated on #{Time.now.to_s}"
+      content_type= "Content-Type: #{@content_type}" if @content_type
+    msg = <<END_OF_MESSAGE
+From: #{from_alias} <#{from}>
+To: #{to_alias} <#{@to}>
+Subject: #{subject}
+#{content_type}
+
+#{@data.to_s}
+END_OF_MESSAGE
+
+      unless @options[:debug]
+        Net::SMTP.start(@host) do |smtp|
+          smtp.send_message msg, from, to
+        end
+      end
+
+      return [msg, from, to] 
+    end
+
+    def << string
+      data << string
+    end
+
+    def puts string
+      data << string
+    end
+
+  end
+end