request-log-analyzer 1.0.2

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -0,0 +1,116 @@
+require File.dirname(__FILE__) + '/../tracker/base'
+
+module RequestLogAnalyzer::Aggregator
+
+  class Summarizer < Base
+    
+    class Definer
+      
+      attr_reader :trackers
+      
+      def initialize
+        @trackers = []
+      end
+      
+      def method_missing(tracker_method, *args)
+        track(tracker_method, args.first)
+      end
+      
+      def category(category_field, options = {})
+        if category_field.kind_of?(Symbol)
+          track(:category, options.merge(:category => category_field))
+        elsif category_field.kind_of?(Hash)
+          track(:category, category_field.merge(options))
+        end
+      end
+      
+      def duration(duration_field, options = {})
+        if duration_field.kind_of?(Symbol)
+          track(:duration, options.merge(:duration => duration_field))
+        elsif duration_field.kind_of?(Hash)
+          track(:duration, duration_field.merge(options))        
+        end
+      end      
+      
+      def track(tracker_klass, options = {})
+        require "#{File.dirname(__FILE__)}/../tracker/#{tracker_klass}"
+        tracker_klass = RequestLogAnalyzer::Tracker.const_get(tracker_klass.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')) if tracker_klass.kind_of?(Symbol)
+        @trackers << tracker_klass.new(options)
+      end
+    end
+    
+    attr_reader :trackers
+    attr_reader :warnings_encountered
+    
+    def initialize(source, options = {})
+      super(source, options)
+      @warnings_encountered = {}
+      @trackers = source.file_format.report_trackers
+      setup
+    end
+    
+    def setup
+    end
+    
+    def prepare
+      raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
+      @trackers.each { |tracker| tracker.prepare }
+      end
+    
+    def aggregate(request)
+      @trackers.each do |tracker|
+        tracker.update(request) if tracker.should_update?(request)
+      end
+    end
+    
+    def finalize
+      @trackers.each { |tracker| tracker.finalize }
+    end
+       
+    def report(output=STDOUT, report_width = 80, color = false)
+      report_header(output, report_width, color)
+      if source.parsed_requests > 0
+        @trackers.each { |tracker| tracker.report(output, report_width, color) }
+      else
+        output << "\n"
+        output << "There were no requests analyzed.\n"
+      end
+      report_footer(output, report_width, color)
+    end
+    
+    def report_header(output=STDOUT, report_width = 80, color = false)
+      output << "Request summary\n"
+      output << green("━" * report_width, color) + "\n"
+      output << "Parsed lines:         #{green(source.parsed_lines, color)}\n"
+      output << "Parsed requests:      #{green(source.parsed_requests, color)}\n"
+      output << "Skipped lines:        #{green(source.skipped_lines, color)}\n" if source.skipped_lines > 0
+      if has_warnings?
+        output <<  "Warnings:             " + @warnings_encountered.map { |(key, value)| "#{key.inspect}: #{blue(value, color)}" }.join(', ') + "\n"
+      end
+      output << "\n"
+    end
+    
+    def report_footer(output=STDOUT, report_width = 80, color = false)
+      output << "\n" 
+      if has_serious_warnings?      
+        output << green("━" * report_width, color) + "\n"
+        output << "Multiple warnings were encountered during parsing. Possibly, your logging " + "\n"
+        output << "is not setup correctly. Visit this website for logging configuration tips:" + "\n"
+        output <<  blue("http://github.com/wvanbergen/request-log-analyzer/wikis/configure-logging", color) + "\n"
+      end
+    end
+    
+    def has_warnings?
+       @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
+    end
+    
+    def has_serious_warnings?
+      @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 10
+    end
+    
+    def warning(type, message, lineno)
+      @warnings_encountered[type] ||= 0
+      @warnings_encountered[type] += 1
+    end
+  end
+end

data/lib/request_log_analyzer/controller.rb

@@ -0,0 +1,201 @@
+module RequestLogAnalyzer
+  
+  # The RequestLogAnalyzer::Controller class creates a LogParser instance for the
+  # requested file format, and connect it with sources and aggregators.
+  #
+  # Sources are streams or files from which the requests will be parsed.
+  # Aggregators will handle every passed request to yield a meaningfull results.
+  #
+  # - Use the build-function to build a controller instance using command line arguments.
+  # - Use add_aggregator to register a new aggregator
+  # - Use add_source to register a new aggregator
+  # - Use the run! method to start the parser and send the requests to the aggregators.
+  #
+  # Note that the order of sources can be imported if you have log files than succeed
+  # eachother. Requests that span over succeeding files will be parsed correctly if the
+  # sources are registered in the correct order. This can be helpful to parse requests
+  # from several logrotated log files.
+  class Controller
+
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    attr_reader :aggregators
+    attr_reader :filters
+    attr_reader :log_parser
+    attr_reader :source
+    attr_reader :output
+    attr_reader :options
+
+    # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
+    # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
+    # <rr>report_with</tt> Width of the report. Defaults to 80.
+    def self.build(arguments, report_width = 80)
+            
+      options = { :report_width => arguments[:report_width].to_i, :output => STDOUT}
+
+      options[:database] = arguments[:database] if arguments[:database]
+      options[:debug]    = arguments[:debug]
+      options[:colorize] = !arguments[:boring]
+
+      if arguments[:file]
+        options[:output] = File.new(arguments[:file], "w+")
+        options[:colorize] = false
+      end
+                
+      # Create the controller with the correct file format
+      file_format = RequestLogAnalyzer::FileFormat.load(arguments[:format])
+
+      # register sources
+      if arguments.parameters.length == 1
+        file = arguments.parameters[0]
+        if file == '-' || file == 'STDIN'
+          options.store(:source_files, $stdin)
+        elsif File.exist?(file)
+          options.store(:source_files, file)
+        else
+          puts "File not found: #{file}"
+          exit(0)
+        end
+      else
+        options.store(:source_files, arguments.parameters)
+      end
+      
+      controller = Controller.new(RequestLogAnalyzer::Source::LogFile.new(file_format, options), options)
+
+      options[:assume_correct_order] = arguments[:assume_correct_order]
+      
+      # register filters        
+      if arguments[:after] || arguments[:before]
+        filter_options = {}
+        filter_options[:after]  = DateTime.parse(arguments[:after])  
+        filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
+        controller.add_filter(:timespan, filter_options)
+      end
+      
+      arguments[:reject].each do |(field, value)|
+        controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
+      end
+      
+      arguments[:select].each do |(field, value)|
+        controller.add_filter(:field, :mode => :select, :field => field, :value => value)
+      end
+
+      # register aggregators
+      arguments[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
+
+      # register the database 
+      controller.add_aggregator(:database)   if arguments[:database] && !arguments[:aggregator].include?('database')
+      controller.add_aggregator(:summarizer) if arguments[:aggregator].empty?
+    
+      # register the echo aggregator in debug mode
+      controller.add_aggregator(:echo) if arguments[:debug]
+      
+      file_format.setup_environment(controller)
+          
+      return controller
+    end
+
+    # Builds a new Controller for the given log file format.
+    # <tt>format</tt> Logfile format. Defaults to :rails
+    # Options are passd on to the LogParser.
+    # * <tt>:aggregator</tt> Aggregator array.
+    # * <tt>:database</tt> Database the controller should use.
+    # * <tt>:echo</tt> Output debug information.
+    # * <tt>:silent</tt> Do not output any warnings.
+    # * <tt>:colorize</tt> Colorize output
+    # * <tt>:output</tt> All report outputs get << through this output.
+    def initialize(source, options = {})
+
+      @source      = source
+      @options     = options
+      @aggregators = []
+      @filters     = []
+      @output      = options[:output]
+      
+      # Requester format through RequestLogAnalyzer::FileFormat and construct the parser
+      register_file_format(@source.file_format) 
+      
+      # Pass all warnings to every aggregator so they can do something useful with them.
+      @source.warning = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } } if @source
+
+      # Handle progress messagess
+      @source.progress = lambda { |message, value| handle_progress(message, value) } if @source
+    end
+    
+    # Progress function.
+    # Expects :started with file, :progress with current line and :finished or :interrupted when done.
+    # <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
+    # <tt>value</tt> File or current line.
+    def handle_progress(message, value = nil)
+      case message
+      when :started
+        @progress_bar = ProgressBar.new(green(File.basename(value), options[:colorize]), File.size(value))
+      when :finished
+        @progress_bar.finish
+        @progress_bar = nil
+      when :interrupted
+        if @progress_bar
+          @progress_bar.halt
+          @progress_bar = nil
+        end
+      when :progress
+        @progress_bar.set(value)
+      end
+    end
+    
+    # Adds an aggregator to the controller. The aggregator will be called for every request 
+    # that is parsed from the provided sources (see add_source)
+    def add_aggregator(agg)
+      if agg.kind_of?(Symbol)
+        require File.dirname(__FILE__) + "/aggregator/#{agg}"
+        agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
+      end
+      
+      @aggregators << agg.new(@source, @options)
+    end
+    
+    alias :>> :add_aggregator
+    
+    # Adds a request filter to the controller.
+    def add_filter(filter, filter_options = {})
+      if filter.kind_of?(Symbol)
+        require File.dirname(__FILE__) + "/filter/#{filter}"
+        filter = RequestLogAnalyzer::Filter.const_get(filter.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
+      end
+      
+      @filters << filter.new(file_format, @options.merge(filter_options))
+    end
+    
+    # Runs RequestLogAnalyzer
+    # 1. Call prepare on every aggregator
+    # 2. Generate requests from source object
+    # 3. Filter out unwanted requests
+    # 4. Call aggregate for remaning requests on every aggregator
+    # 4. Call finalize on every aggregator
+    # 5. Call report on every aggregator
+    # 6. Finalize Source
+    def run!
+      
+      @filters.each { |filter| filter.prepare }
+      @aggregators.each { |agg| agg.prepare }
+      
+      begin
+        @source.requests do |request|
+          #@filters.each { |filter| request = filter.filter(request) }
+          @aggregators.each { |agg| agg.aggregate(request) } if request
+        end
+      rescue Interrupt => e
+        handle_progress(:interrupted)
+        puts "Caught interrupt! Stopped parsing."
+      end
+
+      puts "\n"
+      
+      @aggregators.each { |agg| agg.finalize }
+      @aggregators.each { |agg| agg.report(@output, options[:report_width], options[:colorize]) }
+      
+      @source.finalize
+    end
+    
+  end
+end

data/lib/request_log_analyzer/file_format.rb

@@ -0,0 +1,81 @@
+module RequestLogAnalyzer
+  
+  class FileFormat
+    
+    # Makes classes aware of a file format by registering the file_format variable
+    module Awareness
+      
+      def self.included(base)
+        base.send(:attr_reader, :file_format)
+      end
+      
+      def register_file_format(format)
+        @file_format = format
+      end
+    end
+      
+    def self.inherited(subclass)
+       subclass.instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
+       subclass.class_eval { class << self; attr_accessor :line_definer; end } 
+       subclass.class_eval { class << self; attr_accessor :report_definer; end }       
+    end    
+    
+    def self.line_definition(name, &block)
+      @line_definer.send(name, &block)
+    end
+    
+    def self.format_definition(&block)
+      if block_given?
+        yield(@line_definer) 
+      else
+        return @line_definer
+      end
+    end
+    
+    def self.report(&block)
+      @report_definer = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
+      yield(@report_definer)
+    end
+
+    def self.load(file_format)
+      if file_format.kind_of?(RequestLogAnalyzer::FileFormat)
+        # this already is a file format! return itself
+        return file_format
+
+      elsif file_format.kind_of?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::FileFormat)
+        klass = file_format
+
+      elsif file_format.kind_of?(String) && File.exist?(file_format)
+        # load a format from a ruby file
+        require file_format
+        klass_name = File.basename(file_format, '.rb').split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
+        klass = Object.const_get(klass_name)
+
+      elsif File.exist?("#{File.dirname(__FILE__)}/file_format/#{file_format}.rb")
+        # load a provided file format
+        require "#{File.dirname(__FILE__)}/file_format/#{file_format}"
+        klass_name = file_format.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
+        klass = RequestLogAnalyzer::FileFormat.const_get(klass_name)
+        
+      end
+      
+      klass.new # return an instance of the class
+    end
+    
+    def line_definitions
+      @line_definitions ||= self.class.line_definer.line_definitions
+    end
+    
+    def report_trackers
+      self.class.instance_variable_get(:@report_definer).trackers rescue []
+    end
+    
+    def valid?
+      line_definitions.detect { |(name, ld)| ld.header } && line_definitions.detect { |(name, ld)| ld.footer }
+    end
+    
+    def setup_environment(controller)
+      
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/merb.rb

@@ -0,0 +1,33 @@
+module RequestLogAnalyzer::FileFormat::Merb
+
+  LINE_DEFINITIONS = {
+    
+    # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+    :started => {
+      :header   => true,
+      :teaser   => /Started/,
+      :regexp   => /Started request handling\:\ (.+)/,
+      :captures => [{ :name => :timestamp, :type => :timestamp, :anonymize => :slightly }]
+    },
+    
+    # ~ Params: {"action"=>"create", "controller"=>"session"}
+    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
+    :params => {
+      :teaser   => /Params/,
+      :regexp   => /Params\:\ \{(.+)\}/,
+      :captures => [{ :name => :raw_hash, :type => :string}]
+    },
+    
+    # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
+    :completed => {
+      :footer   => true,
+      :teaser   => /\{:dispatch_time/,
+      :regexp   => /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/,
+      :captures => [{ :name => :dispatch_time,       :type => :sec, :anonymize => :slightly }, 
+                    { :name => :after_filters_time,  :type => :sec, :anonymize => :slightly }, 
+                    { :name => :before_filters_time, :type => :sec, :anonymize => :slightly }, 
+                    { :name => :action_time,         :type => :sec, :anonymize => :slightly }]
+    }
+  }
+    
+end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -0,0 +1,90 @@
+class RequestLogAnalyzer::FileFormat::Rails < RequestLogAnalyzer::FileFormat
+  
+  # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
+  line_definition :processing do |line|
+    line.header = true # this line is the first log line for a request 
+    line.teaser = /Processing /
+    line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
+    line.captures << { :name => :controller, :type  => :string } \
+                  << { :name => :action,     :type  => :string } \
+                  << { :name => :format,     :type  => :string } \
+                  << { :name => :ip,         :type  => :string, :anonymize => :ip } \
+                  << { :name => :timestamp,  :type  => :timestamp, :anonymize => :slightly } \
+                  << { :name => :method,     :type  => :string }
+  end
+
+  # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.    
+  line_definition :cache_hit do |line|
+    line.regexp = /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/
+  end
+  
+  # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
+  line_definition :failed do |line|
+    line.footer = true
+    line.regexp = /((?:[A-Z]\w+\:\:)*[A-Z]\w+) \((.*)\)(?: on line #(\d+) of .+)?\:(.*)/
+    line.captures << { :name => :error,       :type => :string } \
+                  << { :name => :message,     :type => :string } \
+                  << { :name => :line,        :type => :integer } \
+                  << { :name => :file,        :type => :string } \
+                  << { :name => :stack_trace, :type => :string, :anonymize => true }
+  end
+
+
+  # Rails < 2.1 completed line example
+  # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+  RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
+
+  # Rails > 2.1 completed line example
+  # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
+  RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
+
+  # The completed line uses a kind of hack to ensure that both old style logs and new style logs 
+  # are both parsed by the same regular expression. The format in Rails 2.2 was slightly changed,
+  # but the line contains exactly the same information.
+  line_definition :completed do |line|
+    
+    line.footer = true
+    line.teaser = /Completed in /
+    line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
+    
+    line.captures << { :name => :duration, :type => :sec,    :anonymize => :slightly } \
+                  << { :name => :view,     :type => :sec,    :anonymize => :slightly } \
+                  << { :name => :db,       :type => :sec,    :anonymize => :slightly } \
+                  << { :name => :status,   :type => :integer } \
+                  << { :name => :url,      :type => :string, :anonymize => :url } # Old variant 
+                  
+    line.captures << { :name => :duration, :type => :msec,   :anonymize => :slightly } \
+                  << { :name => :view,     :type => :msec,   :anonymize => :slightly } \
+                  << { :name => :db,       :type => :msec,   :anonymize => :slightly } \
+                  << { :name => :status,   :type => :integer} \
+                  << { :name => :url,      :type => :string, :anonymize => :url }  # 2.2 variant 
+  end
+
+
+
+  REQUEST_CATEGORIZER = Proc.new do |request|
+    format = request[:format] || 'html'
+    "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+  end
+
+  report do |analyze|
+    analyze.timespan :line_type => :processing
+    analyze.category :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
+    analyze.category :method, :title => 'HTTP methods'
+    analyze.category :status, :title => 'HTTP statuses returned'
+    analyze.category :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
+    
+    analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
+    analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
+    analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+    
+    analyze.category :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
+            :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
+            
+    analyze.hourly_spread :line_type => :processing
+    analyze.category :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+  end
+
+
+
+end