request-log-analyzer 1.0.2 → 1.6.3

data/lib/request_log_analyzer/line_definition.rb

@@ -1,30 +1,22 @@
 module RequestLogAnalyzer
-  
-  module Anonymizers
-    def anonymizer_for_ip(value, capture_definition)
-      '127.0.0.1'
-    end
 
-    def anonymizer_for_url(value, capture_definition)
-      value.sub(/^https?\:\/\/[A-z0-9\.-]+\//, "http://example.com/")
-    end
-  end  
-  
   # The line definition class is used to specify what lines should be parsed from the log file.
   # It contains functionality to match a line against the definition and parse the information
   # from this line. This is used by the LogParser class when parsing a log file..
   class LineDefinition
 
-    include RequestLogAnalyzer::Anonymizers
-
     class Definer
-      
+
       attr_accessor :line_definitions
-      
+
       def initialize
         @line_definitions = {}
       end
-      
+
+      def initialize_copy(other)
+        @line_definitions = other.line_definitions.dup
+      end
+
       def method_missing(name, *args, &block)
         if block_given?
           @line_definitions[name] = RequestLogAnalyzer::LineDefinition.define(name, &block)
@@ -34,126 +26,107 @@ module RequestLogAnalyzer
       end
     end
 
+    class CaptureDefiner
+      attr_accessor :capture_hash
+      
+      def initialize(hash)
+        @capture_hash = hash
+      end
+      
+      def as(type, type_options = {})
+        @capture_hash.merge!(type_options.merge(:type => type))
+        return self
+      end
+    end
+
     attr_reader :name
     attr_accessor :teaser, :regexp, :captures
     attr_accessor :header, :footer
-    
+
+    alias_method :header?, :header
+    alias_method :footer?, :footer
+
     # Initializes the LineDefinition instance with a hash containing the different elements of
     # the definition.
     def initialize(name, definition = {})
       @name     = name
       @captures = []
+      @teaser   = nil
       definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
     end
-    
+
     def self.define(name, &block)
       definition = self.new(name)
       yield(definition) if block_given?
       return definition
     end
     
-    # Converts a parsed value (String) to the desired value using some heuristics.
-    def convert_value(value, type)
-      case type
-      when :integer;   value.to_i
-      when :float;     value.to_f
-      when :decimal;   value.to_f          
-      when :symbol;    value.to_sym
-      when :sec;       value.to_f
-      when :msec;      value.to_f / 1000
-      when :timestamp; value.gsub(/[^0-9]/,'')[0..13].to_i # Retrieve with: DateTime.parse(value, '%Y%m%d%H%M%S')
-      else value
-      end
+    def capture(name)
+      new_capture_hash = { :name => name, :type => :string}
+      captures << new_capture_hash
+      CaptureDefiner.new(new_capture_hash)
     end
-    
-    # Checks whether a given line matches this definition. 
+
+    # Checks whether a given line matches this definition.
     # It will return false if a line does not match. If the line matches, a hash is returned
     # with all the fields parsed from that line as content.
     # If the line definition has a teaser-check, a :teaser_check_failed warning will be emitted
     # if this teaser-check is passed, but the full regular exprssion does not ,atch.
-    def matches(line, lineno = nil, parser = nil)
+    def matches(line, &warning_handler)
       if @teaser.nil? || @teaser =~ line
         if match_data = line.match(@regexp)
-          request_info = { :line_type => name, :lineno => lineno }
-          
-          captures.each_with_index do |capture, index|
-            next if capture == :ignore
-
-            if match_data.captures[index]
-              request_info[capture[:name]] = convert_value(match_data.captures[index], capture[:type])
-            end
-
-          end
-          return request_info
+          return { :line_definition => self, :captures => match_data.captures}
         else
-          if @teaser && parser
-            parser.warn(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")
+          if @teaser && warning_handler
+            warning_handler.call(:teaser_check_failed, "Teaser matched for #{name.inspect}, but full line did not:\n#{line.inspect}")
           end
           return false
         end
       else
         return false
       end
+    rescue
+      return false
     end
-    
+
     alias :=~ :matches
-    
-    def anonymize_value(value, capture_definition)
-      if capture_definition[:anonymize].respond_to?(:call)
-        capture_definition[:anonymize].call(value, capture_definition)
+
+    # matches the line and converts the captured values using the request's
+    # convert_value function.
+    def match_for(line, request, &warning_handler)
+      if match_info = matches(line, &warning_handler)
+        convert_captured_values(match_info[:captures], request)
       else
-        case capture_definition[:anonymize]
-        when nil;   value
-        when false; value
-        when true;  '***'
-        when :slightly; anonymize_slightly(value, capture_definition)
-        else 
-          method_name = "anonymizer_for_#{capture_definition[:anonymize]}".to_sym
-          self.respond_to?(method_name) ? self.send(method_name, value, capture_definition) : '***'
-        end
+        false
       end
     end
-    
-    def anonymize_slightly(value, capture_definition)  
-      case capture_definition[:type]
-      when :integer
-        (value.to_i * (0.8 + rand * 0.4)).to_i
-      when :double
-        (value.to_f * (0.8 + rand * 0.4)).to_f          
-      when :msec
-        (value.to_i * (0.8 + rand * 0.4)).to_i
-      when :sec
-        (value.to_f * (0.8 + rand * 0.4)).to_f
-      when :timestamp
-        (DateTime.parse(value) + (rand(100) - 50)).to_s
-      else
-        puts "Cannot anonymize #{capture_definition[:type].inspect} slightly, using ***"
-        '***'
-      end  
-    end
 
-    # Anonymize a log line
-    def anonymize(line, options = {})
-      if self.teaser.nil? || self.teaser =~ line
-        if self.regexp =~ line
-          pos_adjustment = 0
-          captures.each_with_index do |capture, index|
-            unless $~[index + 1].nil?
-              anonymized_value = anonymize_value($~[index + 1], capture).to_s
-              line[($~.begin(index + 1) + pos_adjustment)...($~.end(index + 1) + pos_adjustment)] = anonymized_value
-              pos_adjustment += anonymized_value.length - $~[index + 1].length                    
-            end
+    # Updates a captures hash using the converters specified in the request
+    # and handle the :provides option in the line definition.
+    def convert_captured_values(values, request)
+      value_hash = {}
+      captures.each_with_index do |capture, index|
+
+        # convert the value using the request convert_value function
+        converted = request.convert_value(values[index], capture)
+        value_hash[capture[:name]] ||= converted
+
+        # Add items directly to the resulting hash from the converted value
+        # if it is a hash and they are set in the :provides hash for this line definition
+        if converted.kind_of?(Hash) && capture[:provides].kind_of?(Hash)
+          capture[:provides].each do |name, type|
+            value_hash[name] ||= request.convert_value(converted[name], { :type => type })
           end
-          line
-        elsif self.teaser.nil?
-          nil
-        else
-          options[:discard_teaser_lines] ? "" : line
         end
-      else
-        nil
       end
+      return value_hash
+    end
+
+    # Returns true if this line captures values of the given name
+    def captures?(name)
+      captures.any? { |c| c[:name] == name }
     end
+
   end
-  
+
 end

data/lib/request_log_analyzer/log_processor.rb

@@ -1,58 +1,53 @@
 module RequestLogAnalyzer
-  
+
   # The Logprocessor class is used to perform simple processing actions over log files.
-  # It will go over the log file/stream line by line, pass the line to a processor and 
-  # write the result back to the output file or stream. The processor can alter the 
+  # It will go over the log file/stream line by line, pass the line to a processor and
+  # write the result back to the output file or stream. The processor can alter the
   # contents of the line, remain it intact or remove it altogether, based on the current
   # file format
   #
-  # Currently, two processors are supported, :strip and :anonymize. 
-  #  * :strip will remove all irrelevent lines (according to the file format) from the 
+  # Currently, one processors is supported:
+  #  * :strip will remove all irrelevent lines (according to the file format) from the
   #    sources. A compact, information packed log will remain/.
-  #  * :anonymize will anonymize sensitive information from the lines according to the 
-  #    anonymization rules in the file format. The result can be passed to third parties
-  #    without privacy concerns.
   #
   class LogProcessor
-    
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    attr_reader :mode, :options, :sources
+
+    attr_reader :mode, :options, :sources, :file_format
     attr_accessor :output_file
-   
+
     # Builds a logprocessor instance from the arguments given on the command line
-    # <tt>command</tt> The command hat was used to start the log processor. This can either be 
-    #   :strip or :anonymize. This will set the processing mode.
+    # <tt>command</tt> The command hat was used to start the log processor. This will set the
+    #    processing mode. Currently, only :strip is supported.
     # <tt>arguments</tt> The parsed command line arguments (a CommandLine::Arguments instance)
     def self.build(command, arguments)
-      
-      options = { 
-          :discard_teaser_lines => arguments[:discard_teaser_lines], 
-          :keep_junk_lines      => arguments[:keep_junk_lines], 
+
+      options = {
+          :discard_teaser_lines => arguments[:discard_teaser_lines],
+          :keep_junk_lines      => arguments[:keep_junk_lines],
         }
-          
+
       log_processor = RequestLogAnalyzer::LogProcessor.new(arguments[:format].to_sym, command, options)
       log_processor.output_file = arguments[:output] if arguments[:output]
 
       arguments.parameters.each do |input|
         log_processor.sources << input
       end
-      
+
       return log_processor
     end
-    
+
     # Initializes a new LogProcessor instance.
     # <tt>format</tt> The file format to use (e.g. :rails).
-    # <tt>mode</tt> The processing mode (:anonymize or :strip)
+    # <tt>mode</tt> The processing mode
     # <tt>options</tt> A hash with options to take into account
     def initialize(format, mode, options = {})
-      @options = options
-      @mode    = mode
-      @sources = []
+      @options     = options
+      @mode        = mode
+      @sources     = []
+      @file_format = format
       $output_file = nil
-      self.register_file_format(format)
     end
-    
+
     # Processes input files by opening it and sending the filestream to <code>process_io</code>,
     # in which the actual processing is performed.
     # <tt>file</tt> The file to process
@@ -61,15 +56,14 @@ module RequestLogAnalyzer
     end
 
     # Processes an input stream by iteration over each line and processing it according to
-    # the current operation mode (:strip, :anonymize)
+    # the current operation mode
     # <tt>io</tt> The IO instance to process.
     def process_io(io)
       case mode
         when :strip;     io.each_line { |line| @output << strip_line(line) }
-        when :anonymize; io.each_line { |line| @output << anonymize_line(line) }
       end
     end
-    
+
     # Returns the line itself if the string matches any of the line definitions. If no match is
     # found, an empty line is returned, which will strip the line from the output.
     # <tt>line</tt> The line to strip
@@ -77,42 +71,26 @@ module RequestLogAnalyzer
       file_format.line_definitions.any? { |name, definition| definition =~ line } ? line : ""
     end
 
-    # Returns an anonymized version of the provided line. This can be a copy of the line it self, 
-    # an empty string or a string in which some substrings are substituted for anonymized values.
-    # <tt>line</tt> The line to anonymize
-    def anonymize_line(line)
-      anonymized_line = nil
-      file_format.line_definitions.detect { |name, definition| anonymized_line = definition.anonymize(line, options) }
-      
-      if anonymized_line
-        return anonymized_line
-      elsif options[:keep_junk_lines]
-        return line
-      else
-        return ""
-      end
-    end
-
     # Runs the log processing by setting up the output stream and iterating over all the
-    # input sources. Input sources can either be filenames (String instances) or IO streams 
+    # input sources. Input sources can either be filenames (String instances) or IO streams
     # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable.
     def run!
       if @output_file.nil?
-        @output = $stdout 
+        @output = $stdout
       else
-        @output = File.new(@output_file, 'a')        
+        @output = File.new(@output_file, 'a')
       end
-      
+
       @sources.each do |source|
         if source.kind_of?(String) && File.exist?(source)
           process_file(source)
         elsif source.kind_of?(IO)
           process_io(source)
         elsif ['-', 'STDIN'].include?(source)
-          process_io($stdin)        
+          process_io($stdin)
         end
       end
-    
+
     ensure
       @output.close if @output.kind_of?(File)
     end

data/lib/request_log_analyzer/mailer.rb

@@ -0,0 +1,65 @@
+module RequestLogAnalyzer
+
+  # Mail report to a specified emailaddress
+  class Mailer
+
+    attr_accessor :data, :to, :host, :port
+
+    # Initialize a mailer
+    # <tt>to</tt> to email address to mail to
+    # <tt>host</tt> the mailer host (defaults to localhost)
+    # <tt>options</tt> Specific style options
+    #
+    # Options
+    # * <tt>:debug</tt> Do not actually mail
+    # * <tt>:from_alias</tt> The from alias
+    # * <tt>:to_alias</tt> The to alias
+    # * <tt>:subject</tt> The message subject
+    def initialize(to, host = 'localhost', options = {})
+      require 'net/smtp'
+      @to      = to
+      @host    = host
+
+      @port    = 25
+      @options = options
+      @host, @port = host.split(':') if @host.include?(':')
+      @data    = []      
+    end
+
+    # Send all data in @data to the email address used during initialization.
+    # Returns array containg [message_data, from_email_address, to_email_address] of sent email.
+    def mail
+      from          = @options[:from]        || 'contact@railsdoctors.com'
+      from_alias    = @options[:from_alias]  || 'Request-log-analyzer reporter'
+      to_alias      = @options[:to_alias]    || to
+      subject       = @options[:subject]     || "Request log analyzer report - generated on #{Time.now.to_s}"
+      content_type  = ""
+      content_type  = 'Content-Type: text/html; charset="ISO-8859-1";' if @data.map{|l| l.include?('html')}.include?(true)
+    msg = <<END_OF_MESSAGE
+From: #{from_alias} <#{from}>
+To: #{to_alias} <#{@to}>
+Subject: #{subject}
+#{content_type}
+
+#{@data.join("\n")}
+END_OF_MESSAGE
+
+      unless @options[:debug]
+        Net::SMTP.start(@host, @port) do |smtp|
+          smtp.send_message msg, from, to
+        end
+      end
+
+      return [msg, from, to] 
+    end
+
+    def << string
+      data << string
+    end
+
+    def puts string
+      data << string
+    end
+
+  end
+end

data/lib/request_log_analyzer/output/fancy_html.rb

@@ -0,0 +1,49 @@
+begin 
+  require 'rubygems'
+  require 'gchart'
+rescue LoadError
+  $stderr.puts "The FancyHTML output format requires the googlechart gem:"
+  $stderr.puts "  (sudo) gem install googlecharts"
+end
+
+module RequestLogAnalyzer::Output
+  
+  class FancyHTML < HTML
+    
+    # Load class files if needed
+    def self.const_missing(const)
+      RequestLogAnalyzer::load_default_class_file(self, const)
+    end
+    
+    def report_tracker(tracker)
+      case tracker
+      when RequestLogAnalyzer::Tracker::HourlySpread then report_hourly_spread(tracker)
+      else tracker.report(self)
+      end
+    end
+    
+    def report_hourly_spread(tracker)
+      title tracker.title
+      puts tag(:img, nil, :width => '700', :height => '120', :src =>
+          Gchart.sparkline(:data => tracker.hour_frequencies, :size => '700x120', :line_colors => '0077CC',
+                           :axis_with_labels => 'x,y', :axis_labels => [x_axis_labels(tracker),y_axis_labels(tracker)]))
+    end
+    
+    def x_axis_labels(tracker)
+      frmt = '%H:%M'
+      x_axis_labels = []
+      num_labels    = 5
+      start         = tracker.first_timestamp
+      step          = tracker.timespan / (num_labels - 1)
+      for i in 0...num_labels do
+        x_axis_labels << (start + step * i).strftime(frmt)
+      end
+      x_axis_labels
+    end
+    
+    def y_axis_labels(tracker)
+      sorted_frequencies = tracker.hour_frequencies.sort
+      [sorted_frequencies.first, sorted_frequencies.last]
+    end
+  end
+end