wvanbergen-request-log-analyzer 1.0.0 → 1.0.1

data/lib/request_log_analyzer/controller.rb

@@ -0,0 +1,206 @@
+module RequestLogAnalyzer
+  
+  # The RequestLogAnalyzer::Controller class creates a LogParser instance for the
+  # requested file format, and connect it with sources and aggregators.
+  #
+  # Sources are streams or files from which the requests will be parsed.
+  # Aggregators will handle every passed request to yield a meaningfull results.
+  #
+  # - Use the build-function to build a controller instance using command line arguments.
+  # - Use add_aggregator to register a new aggregator
+  # - Use add_source to register a new aggregator
+  # - Use the run! method to start the parser and send the requests to the aggregators.
+  #
+  # Note that the order of sources can be imported if you have log files than succeed
+  # eachother. Requests that span over succeeding files will be parsed correctly if the
+  # sources are registered in the correct order. This can be helpful to parse requests
+  # from several logrotated log files.
+  class Controller
+
+    include RequestLogAnalyzer::FileFormat::Awareness
+    
+    attr_reader :aggregators
+    attr_reader :filters
+    attr_reader :log_parser
+    attr_reader :source
+    attr_reader :output
+    attr_reader :options
+
+    # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
+    # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
+    # <rr>report_with</tt> Width of the report. Defaults to 80.
+    def self.build(arguments, report_width = 80)
+            
+      options = { :report_width => arguments[:report_width].to_i, :output => STDOUT}
+
+      options[:combined_requests] = !arguments[:single_lines]
+      options[:database] = arguments[:database] if arguments[:database]
+      options[:debug]    = arguments[:debug]
+      options[:colorize] = !arguments[:boring]
+
+      if arguments[:file]
+        options[:output] = File.new(arguments[:file], "w+")
+        options[:colorize] = false
+      end
+                
+      # Create the controller with the correct file format
+      file_format = RequestLogAnalyzer::FileFormat.load(arguments[:format])
+      source_files = nil
+      # register sources
+      if arguments.parameters.length == 1
+        file = arguments.parameters[0]
+        if file == '-' || file == 'STDIN'
+          options.store(:source_files, $stdin)
+        elsif File.exist?(file)
+          options.store(:source_files, file)
+        else
+          puts "File not found: #{file}"
+          exit(0)
+        end
+      else
+        options.store(:source_files, arguments.parameters)
+      end
+      
+      controller = Controller.new(RequestLogAnalyzer::Source::LogFile.new(file_format, options), options)
+
+      # register filters
+      # filters are only supported in combined requests mode
+      if options[:combined_requests]
+        
+        options[:assume_correct_order] = arguments[:assume_correct_order]
+        
+        if arguments[:after] || arguments[:before]
+          filter_options = {}
+          filter_options[:after]  = DateTime.parse(arguments[:after])  
+          filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
+          controller.add_filter(:timespan, filter_options)
+        end
+        
+        arguments[:reject].each do |(field, value)|
+          controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
+        end
+        
+        arguments[:select].each do |(field, value)|
+          controller.add_filter(:field, :mode => :select, :field => field, :value => value)
+        end
+        
+      end
+
+      # register aggregators
+      arguments[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
+
+      # register the database 
+      controller.add_aggregator(:database)   if arguments[:database] && !arguments[:aggregator].include?('database')
+      controller.add_aggregator(:summarizer) if arguments[:aggregator].empty?
+    
+      # register the echo aggregator in debug mode
+      controller.add_aggregator(:echo) if arguments[:debug]
+            
+      return controller
+    end
+
+    # Builds a new Controller for the given log file format.
+    # <tt>format</tt> Logfile format. Defaults to :rails
+    # Options are passd on to the LogParser.
+    # * <tt>:aggregator</tt> Aggregator array.
+    # * <tt>:combined_requests</tt> Combine multiline requests into a single request.
+    # * <tt>:database</tt> Database the controller should use.
+    # * <tt>:echo</tt> Output debug information.
+    # * <tt>:silent</tt> Do not output any warnings.
+    # * <tt>:colorize</tt> Colorize output
+    # * <tt>:output</tt> All report outputs get << through this output.
+    def initialize(source, options = {})
+
+      @source      = source
+      @options     = options
+      @aggregators = []
+      @filters     = []
+      @output      = options[:output]
+      
+      # Requester format through RequestLogAnalyzer::FileFormat and construct the parser
+      register_file_format(@source.file_format) 
+      
+      # Pass all warnings to every aggregator so they can do something useful with them.
+      @source.warning = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } } if @source
+
+      # Handle progress messagess
+      @source.progress = lambda { |message, value| handle_progress(message, value) } if @source
+    end
+    
+    # Progress function.
+    # Expects :started with file, :progress with current line and :finished or :interrupted when done.
+    # <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
+    # <tt>value</tt> File or current line.
+    def handle_progress(message, value = nil)
+      case message
+      when :started
+        @progress_bar = ProgressBar.new(green(File.basename(value), options[:colorize]), File.size(value))
+      when :finished
+        @progress_bar.finish
+        @progress_bar = nil
+      when :interrupted
+        if @progress_bar
+          @progress_bar.halt
+          @progress_bar = nil
+        end
+      when :progress
+        @progress_bar.set(value)
+      end
+    end
+    
+    # Adds an aggregator to the controller. The aggregator will be called for every request 
+    # that is parsed from the provided sources (see add_source)
+    def add_aggregator(agg)
+      if agg.kind_of?(Symbol)
+        require File.dirname(__FILE__) + "/aggregator/#{agg}"
+        agg = RequestLogAnalyzer::Aggregator.const_get(agg.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
+      end
+      
+      @aggregators << agg.new(@source, @options)
+    end
+    
+    alias :>> :add_aggregator
+    
+    # Adds a request filter to the controller.
+    def add_filter(filter, filter_options = {})
+      if filter.kind_of?(Symbol)
+        require File.dirname(__FILE__) + "/filter/#{filter}"
+        filter = RequestLogAnalyzer::Filter.const_get(filter.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join(''))
+      end
+      
+      @filters << filter.new(file_format, @options.merge(filter_options))
+    end
+    
+    # Runs RequestLogAnalyzer
+    # 1. Call prepare on every aggregator
+    # 2. Generate requests from source object
+    # 3. Filter out unwanted requests
+    # 4. Call aggregate for remaning requests on every aggregator
+    # 4. Call finalize on every aggregator
+    # 5. Call report on every aggregator
+    # 6. Finalize Source
+    def run!
+      
+      @filters.each { |filter| filter.prepare }
+      @aggregators.each { |agg| agg.prepare }
+      
+      begin
+        @source.requests do |request|
+          @filters.each { |filter| request = filter.filter(request) }
+          @aggregators.each { |agg| agg.aggregate(request) } if request
+        end
+      rescue Interrupt => e
+        handle_progress(:interrupted)
+        puts "Caught interrupt! Stopped parsing."
+      end
+
+      puts "\n"
+      
+      @aggregators.each { |agg| agg.finalize }
+      @aggregators.each { |agg| agg.report(@output, options[:report_width], options[:colorize]) }
+      
+      @source.finalize
+    end
+    
+  end
+end

data/lib/request_log_analyzer/file_format/merb.rb

@@ -0,0 +1,33 @@
+module RequestLogAnalyzer::FileFormat::Merb
+
+  LINE_DEFINITIONS = {
+    
+    # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+    :started => {
+      :header   => true,
+      :teaser   => /Started/,
+      :regexp   => /Started request handling\:\ (.+)/,
+      :captures => [{ :name => :timestamp, :type => :timestamp, :anonymize => :slightly }]
+    },
+    
+    # ~ Params: {"action"=>"create", "controller"=>"session"}
+    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
+    :params => {
+      :teaser   => /Params/,
+      :regexp   => /Params\:\ \{(.+)\}/,
+      :captures => [{ :name => :raw_hash, :type => :string}]
+    },
+    
+    # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
+    :completed => {
+      :footer   => true,
+      :teaser   => /\{:dispatch_time/,
+      :regexp   => /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/,
+      :captures => [{ :name => :dispatch_time,       :type => :sec, :anonymize => :slightly }, 
+                    { :name => :after_filters_time,  :type => :sec, :anonymize => :slightly }, 
+                    { :name => :before_filters_time, :type => :sec, :anonymize => :slightly }, 
+                    { :name => :action_time,         :type => :sec, :anonymize => :slightly }]
+    }
+  }
+    
+end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -0,0 +1,119 @@
+class RequestLogAnalyzer::FileFormat::Rails < RequestLogAnalyzer::FileFormat
+  
+  # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
+  line_definition :processing do |line|
+    line.header = true # this line is the first log line for a request 
+    line.teaser = /Processing /
+    line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
+    line.captures << { :name => :controller, :type  => :string } \
+                  << { :name => :action,     :type  => :string } \
+                  << { :name => :format,     :type  => :string } \
+                  << { :name => :ip,         :type  => :string, :anonymize => :ip } \
+                  << { :name => :timestamp,  :type  => :timestamp, :anonymize => :slightly } \
+                  << { :name => :method,     :type  => :string }
+  end
+
+  # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.    
+  line_definition :cache_hit do |line|
+    line.regexp = /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/
+  end
+  
+  # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
+  line_definition :failed do |line|
+    line.footer = true
+    line.regexp = /((?:[A-Z]\w+\:\:)*[A-Z]\w+) \((.*)\)(?: on line #(\d+) of .+)?\:(.*)/
+    line.captures << { :name => :error,            :type => :string } \
+                  << { :name => :exception_string, :type => :string } \
+                  << { :name => :line,             :type => :integer } \
+                  << { :name => :file,             :type => :string } \
+                  << { :name => :stack_trace,      :type => :string, :anonymize => true }
+  end
+
+
+  # Rails < 2.1 completed line example
+  # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+  RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
+
+  # Rails > 2.1 completed line example
+  # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
+  RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
+
+  # The completed line uses a kind of hack to ensure that both old style logs and new style logs 
+  # are both parsed by the same regular expression. The format in Rails 2.2 was slightly changed,
+  # but the line contains exactly the same information.
+  line_definition :completed do |line|
+    
+    line.footer = true
+    line.teaser = /Completed in /
+    line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
+    
+    line.captures << { :name => :duration, :type => :sec,    :anonymize => :slightly } \
+                  << { :name => :view,     :type => :sec,    :anonymize => :slightly } \
+                  << { :name => :db,       :type => :sec,    :anonymize => :slightly } \
+                  << { :name => :status,   :type => :integer } \
+                  << { :name => :url,      :type => :string, :anonymize => :url } # Old variant 
+                  
+    line.captures << { :name => :duration, :type => :msec,   :anonymize => :slightly } \
+                  << { :name => :view,     :type => :msec,   :anonymize => :slightly } \
+                  << { :name => :db,       :type => :msec,   :anonymize => :slightly } \
+                  << { :name => :status,   :type => :integer} \
+                  << { :name => :url,      :type => :string, :anonymize => :url }  # 2.2 variant 
+  end
+
+
+
+  REQUEST_CATEGORIZER = Proc.new do |request|
+    if request.combined?
+    
+      if request =~ :failed
+        format = request[:format] || 'html'
+        "#{request[:error]} in #{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+      else
+        format = request[:format] || 'html'
+        "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+      end
+    
+    else
+      case request.line_type
+      when :processing   
+        format = request[:format] || 'html'
+        "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
+      
+      when :completed
+        url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
+        url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
+
+        url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date/') # Combine all (year-month-day) queries
+        url.gsub!(/\/\d+-\d+(\/|$)/, '/:month/') # Combine all date (year-month) queries
+        url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs request[:url] # TODO: improve me
+        url
+      
+      when :failed
+        request[:error]
+      else
+        raise "Cannot group this request: #{request.inspect}" 
+      end
+    end
+  end
+
+  report do |analyze|
+    analyze.timespan :line_type => :processing
+    analyze.category :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
+    analyze.category :method, :title => 'HTTP methods'
+    analyze.category :status, :title => 'HTTP statuses returned'
+    analyze.category :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
+    
+    analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
+    analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
+    analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+    
+    analyze.category :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
+            :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
+            
+    analyze.hourly_spread :line_type => :processing
+    analyze.category :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
+  end
+
+
+
+end 

data/lib/request_log_analyzer/file_format.rb

@@ -0,0 +1,77 @@
+module RequestLogAnalyzer
+  
+  class FileFormat
+    
+    # Makes classes aware of a file format by registering the file_format variable
+    module Awareness
+      
+      def self.included(base)
+        base.send(:attr_reader, :file_format)
+      end
+      
+      def register_file_format(format)
+        @file_format = format
+      end
+    end
+      
+    def self.inherited(subclass)
+       subclass.instance_variable_set(:@line_definer, RequestLogAnalyzer::LineDefinition::Definer.new)
+       subclass.class_eval { class << self; attr_accessor :line_definer; end } 
+       subclass.class_eval { class << self; attr_accessor :report_definer; end }       
+    end    
+    
+    def self.line_definition(name, &block)
+      @line_definer.send(name, &block)
+    end
+    
+    def self.format_definition(&block)
+      if block_given?
+        yield(@line_definer) 
+      else
+        return @line_definer
+      end
+    end
+    
+    def self.report(&block)
+      @report_definer = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
+      yield(@report_definer)
+    end
+
+    def self.load(file_format)
+      if file_format.kind_of?(RequestLogAnalyzer::FileFormat)
+        # this already is a file format! return itself
+        return file_format
+
+      elsif file_format.kind_of?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::FileFormat)
+        klass = file_format
+
+      elsif file_format.kind_of?(String) && File.exist?(file_format)
+        # load a format from a ruby file
+        require file_format
+        klass_name = File.basename(file_format, '.rb').split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
+        klass = Object.const_get(klass_name)
+
+      elsif File.exist?("#{File.dirname(__FILE__)}/file_format/#{file_format}.rb")
+        # load a provided file format
+        require "#{File.dirname(__FILE__)}/file_format/#{file_format}"
+        klass_name = file_format.to_s.split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
+        klass = RequestLogAnalyzer::FileFormat.const_get(klass_name)
+        
+      end
+      
+      klass.new # return an instance of the class
+    end
+    
+    def line_definitions
+      @line_definitions ||= self.class.line_definer.line_definitions
+    end
+    
+    def report_trackers
+      self.class.instance_variable_get(:@report_definer).trackers rescue []
+    end
+    
+    def valid?
+      line_definitions.detect { |(name, ld)| ld.header } && line_definitions.detect { |(name, ld)| ld.footer }
+    end
+  end
+end

data/lib/request_log_analyzer/filter/base.rb

@@ -0,0 +1,29 @@
+module RequestLogAnalyzer
+  module Filter
+    # Base filter class used to filter input requests.
+    # All filters should interit from this base.
+    class Base
+      
+      include RequestLogAnalyzer::FileFormat::Awareness
+      
+      attr_reader :log_parser
+      attr_reader :options
+      
+      # Initializer
+      # <tt>format</tt> The file format
+      # <tt>options</tt> Are passed to the filters.
+      def initialize(format, options = {})
+        @options    = options
+        register_file_format(format)
+      end
+      
+      def prepare
+      end
+      
+      def filter(request)
+        return nil unless request
+        return request
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/filter/field.rb

@@ -0,0 +1,36 @@
+module RequestLogAnalyzer::Filter
+  
+  # Filter to select or reject a specific field
+  # Options
+  # * <tt>:mode</tt> :reject or :accept.
+  # * <tt>:field</tt> Specific field to accept or reject.
+  # * <tt>:value</tt> Value that the field should match to be accepted or rejected.
+  class Field < Base
+   
+    attr_reader :field, :value, :mode
+   
+    def prepare
+      @mode = (@options[:mode] || :accept).to_sym
+      @field = @options[:field].to_sym
+      
+      # Convert the timestamp to the correct formats for quick timestamp comparisons
+      if @options[:value].kind_of?(String) && @options[:value][0, 1] == '/' && @options[:value][-1, 1] == '/'
+        @value = Regexp.new(@options[:value][1..-2])
+      else
+        @value = @options[:value] # TODO: convert value?
+      end
+    end
+    
+    def filter(request)
+      return nil unless request
+      
+      found_field = request.every(@field).any? { |value| @value === value }
+      
+      return nil if !found_field && @mode == :select
+      return nil if found_field && @mode == :reject
+      
+      return request
+    end 
+  end
+  
+end

data/lib/request_log_analyzer/filter/timespan.rb

@@ -0,0 +1,32 @@
+module RequestLogAnalyzer::Filter
+  
+  # Reject all requests not in given timespan
+  # Options
+  # * <tt>:after</tt> Only keep requests after this DateTime.
+  # * <tt>:before</tt> Only keep requests before this DateTime.
+  class Timespan < Base
+   
+    attr_reader :before, :after
+   
+    def prepare
+      # Convert the timestamp to the correct formats for quick timestamp comparisons
+      @after  = @options[:after].strftime('%Y%m%d%H%M%S').to_i  if options[:after]     
+      @before = @options[:before].strftime('%Y%m%d%H%M%S').to_i if options[:before]
+    end
+    
+    def filter(request)
+      return nil unless request
+      
+      if @after && @before && request.timestamp <= @before && @after <= request.timestamp
+        return request
+      elsif @after && @before.nil? && @after <= request.timestamp
+        return request
+      elsif @before && @after.nil? && request.timestamp <= @before 
+        return request
+      end
+
+      return nil
+    end 
+  end
+  
+end