request-log-analyzer 1.0.2 → 1.6.3

data/lib/request_log_analyzer/log_parser.rb

@@ -1,173 +0,0 @@
-module RequestLogAnalyzer
-  
-  # The LogParser class reads log data from a given source and uses a file format definition
-  # to parse all relevent information about requests from the file. A FileFormat module should 
-  # be provided that contains the definitions of the lines that occur in the log data.
-  #
-  # De order in which lines occur is used to combine lines to a single request. If these lines 
-  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
-  # the log file simultaneously by different mongrel processes. This problem is detected by the 
-  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
-  # occurs.
-  class LogParser
-    
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    # A hash of options
-    attr_reader :options
-    
-    # The current Request object that is being parsed
-    attr_reader :current_request
-    
-    # The total number of parsed lines
-    attr_reader :parsed_lines
-    
-    # The total number of parsed requests.
-    attr_reader :parsed_requests
-    
-    # The number of skipped requests because of date constraints
-    attr_reader :skipped_requests
-    
-    # Initializes the parser instance.
-    # It will apply the language specific FileFormat module to this instance. It will use the line
-    # definitions in this module to parse any input.
-    def initialize(format, options = {})      
-      @line_definitions = {}
-      @options          = options
-      @parsed_lines     = 0
-      @parsed_requests  = 0
-      @skipped_requests = 0      
-      
-      @current_io = nil
-      
-      # install the file format module (see RequestLogAnalyzer::FileFormat)
-      # and register all the line definitions to the parser
-      self.register_file_format(format)
-    end
-    
-    # Parses a list of consequent files of the same format
-    def parse_files(files, options = {}, &block)
-      files.each { |file| parse_file(file, options, &block) }
-    end
-    
-    # Parses a file. 
-    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
-    def parse_file(file, options = {}, &block)
-      @progress_handler.call(:started, file) if @progress_handler
-      File.open(file, 'r') { |f| parse_io(f, options, &block) }
-      @progress_handler.call(:finished, file) if @progress_handler
-    end
-    
-    def parse_stream(stream, options = {}, &block)
-      parse_io(stream, options, &block)
-    end
-
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def parse_io(io, options = {}, &block)
-
-      # parse every line type by default
-      line_types = options[:line_types] || file_format.line_definitions.keys
-
-      # check whether all provided line types are valid
-      unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
-      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
-      
-      @current_io = io
-      @current_io.each_line do |line|
-        
-        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
-        
-        request_data = nil
-        line_types.each do |line_type|
-          line_type_definition = file_format.line_definitions[line_type]
-          break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
-        end
-
-        if request_data
-          @parsed_lines += 1
-          update_current_request(request_data, &block)
-        end
-      end
-      
-      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
-  
-      @current_io = nil
-    end
-    
-    # Add a block to this method to install a progress handler while parsing
-    def progress=(proc)
-      @progress_handler = proc
-    end
-    
-    # Add a block to this method to install a warning handler while parsing
-    def warning=(proc)
-      @warning_handler = proc
-    end
-    
-    # This method is called by the parser if it encounteres any problems.
-    # It will call the warning handler. The default controller will pass all warnings to every
-    # aggregator that is registered and running
-    def warn(type, message)
-      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
-    end
-    
-    protected
-    
-    # Combines the different lines of a request into a single Request object. It will start a 
-    # new request when a header line is encountered en will emit the request when a footer line 
-    # is encountered.
-    #
-    # - Every line that is parsed before a header line is ignored as it cannot be included in 
-    #   any request. It will emit a :no_current_request warning.
-    # - A header line that is parsed before a request is closed by a footer line, is a sign of
-    #   an unprpertly ordered file. All data that is gathered for the request until then is 
-    #   discarded, the next request is ignored as well and a :unclosed_request warning is
-    #   emitted.
-    def update_current_request(request_data, &block)
-      if header_line?(request_data)
-        unless @current_request.nil?
-          if options[:assume_correct_order]
-            handle_request(@current_request, &block)            
-            @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
-          else
-            warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
-            @current_request = nil # remove all data that was parsed, skip next request as well.
-          end
-        else
-          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
-        end
-      else
-        unless @current_request.nil?
-          @current_request << request_data
-          if footer_line?(request_data)
-            handle_request(@current_request, &block)
-            @current_request = nil 
-          end
-        else
-          warn(:no_current_request, "Parsebale line found outside of a request!")
-        end
-      end
-    end
-    
-    # Handles the parsed request by calling the request handler.
-    # The default controller will send the request to every running aggegator.
-    def handle_request(request, &block)
-      @parsed_requests += 1
-      accepted = block_given? ? yield(request) : true
-      @skipped_requests += 1 if !accepted
-    end
-    
-    # Checks whether a given line hash is a header line.
-    def header_line?(hash)
-      file_format.line_definitions[hash[:line_type]].header
-    end
-
-    # Checks whether a given line hash is a footer line.    
-    def footer_line?(hash)
-      file_format.line_definitions[hash[:line_type]].footer  
-    end 
-  end
-end

data/lib/request_log_analyzer/source/base.rb

@@ -1,42 +0,0 @@
-module RequestLogAnalyzer::Source
-  class Base
-    
-    include RequestLogAnalyzer::FileFormat::Awareness
-
-    # A hash of options
-    attr_reader :options
-
-    # The current Request object that is being parsed
-    attr_reader :current_request
-
-    # The total number of parsed lines
-    attr_reader :parsed_lines
-
-    # The total number of parsed requests.
-    attr_reader :parsed_requests
-
-    # The number of skipped lines because of warnings
-    attr_reader :skipped_lines
-
-    # Base source class used to filter input requests.
-
-    # Initializer
-    # <tt>format</tt> The file format
-    # <tt>options</tt> Are passed to the filters.
-    def initialize(format, options = {})
-      @options    = options
-      register_file_format(format)
-    end
-    
-    def prepare
-    end
-    
-    def requests(&block)
-      return true
-    end
-
-    def finalize
-    end
-
-  end
-end

data/lib/request_log_analyzer/source/log_file.rb

@@ -1,170 +0,0 @@
-module RequestLogAnalyzer::Source
-  
-  # The LogParser class reads log data from a given source and uses a file format definition
-  # to parse all relevent information about requests from the file. A FileFormat module should 
-  # be provided that contains the definitions of the lines that occur in the log data.
-  #
-  # De order in which lines occur is used to combine lines to a single request. If these lines 
-  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
-  # the log file simultaneously by different mongrel processes. This problem is detected by the 
-  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
-  # occurs.
-  class LogFile < RequestLogAnalyzer::Source::Base
-
-    attr_reader :source_files
-
-    # Initializes the parser instance.
-    # It will apply the language specific FileFormat module to this instance. It will use the line
-    # definitions in this module to parse any input.
-    def initialize(format, options = {})      
-      @line_definitions = {}
-      @options          = options
-      @parsed_lines     = 0
-      @parsed_requests  = 0
-      @skipped_lines    = 0
-      @current_io       = nil
-      @source_files     = options[:source_files]
-
-      # install the file format module (see RequestLogAnalyzer::FileFormat)
-      # and register all the line definitions to the parser
-      self.register_file_format(format)
-    end
-    
-    def requests(options = {}, &block)
-      
-      case @source_files
-      when IO;     
-        puts "Parsing from the standard input. Press CTRL+C to finish."
-        parse_stream(@source_files, options, &block) 
-      when String
-        parse_file(@source_files, options, &block) 
-      when Array
-        parse_files(@source_files, options, &block) 
-      else
-        raise "Unknown source provided"
-      end
-    end
-
-    # Parses a list of consequent files of the same format
-    def parse_files(files, options = {}, &block)
-      files.each { |file| parse_file(file, options, &block) }
-    end
-
-    # Parses a file. 
-    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
-    def parse_file(file, options = {}, &block)
-      @progress_handler.call(:started, file) if @progress_handler
-      File.open(file, 'r') { |f| parse_io(f, options, &block) }
-      @progress_handler.call(:finished, file) if @progress_handler
-    end
-
-    def parse_stream(stream, options = {}, &block)
-      parse_io(stream, options, &block)
-    end
-
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def parse_io(io, options = {}, &block)
-
-      # parse every line type by default
-      line_types = options[:line_types] || file_format.line_definitions.keys
-
-      # check whether all provided line types are valid
-      unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
-      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
-
-      @current_io = io
-      @current_io.each_line do |line|
-
-        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
-
-        request_data = nil
-        line_types.each do |line_type|
-          line_type_definition = file_format.line_definitions[line_type]
-          break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
-        end
-
-        if request_data
-          @parsed_lines += 1
-          update_current_request(request_data, &block)
-        end
-      end
-
-      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
-
-      @current_io = nil
-    end
-
-    # Add a block to this method to install a progress handler while parsing
-    def progress=(proc)
-      @progress_handler = proc
-    end
-
-    # Add a block to this method to install a warning handler while parsing
-    def warning=(proc)
-      @warning_handler = proc
-    end
-
-    # This method is called by the parser if it encounteres any problems.
-    # It will call the warning handler. The default controller will pass all warnings to every
-    # aggregator that is registered and running
-    def warn(type, message)
-      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
-    end
-
-    protected
-
-    # Combines the different lines of a request into a single Request object. It will start a 
-    # new request when a header line is encountered en will emit the request when a footer line 
-    # is encountered.
-    #
-    # - Every line that is parsed before a header line is ignored as it cannot be included in 
-    #   any request. It will emit a :no_current_request warning.
-    # - A header line that is parsed before a request is closed by a footer line, is a sign of
-    #   an unprpertly ordered file. All data that is gathered for the request until then is 
-    #   discarded, the next request is ignored as well and a :unclosed_request warning is
-    #   emitted.
-    def update_current_request(request_data, &block)
-      if header_line?(request_data)
-        unless @current_request.nil?
-          if options[:assume_correct_order]
-            @parsed_requests += 1
-            yield @current_request
-            @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
-          else
-            @skipped_lines += 1
-            warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
-            @current_request = nil # remove all data that was parsed, skip next request as well.
-          end
-        else
-          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
-        end
-      else
-        unless @current_request.nil?
-          @current_request << request_data
-          if footer_line?(request_data)
-            @parsed_requests += 1
-            yield @current_request
-            @current_request = nil 
-          end
-        else
-          @skipped_lines += 1
-          warn(:no_current_request, "Parsebale line found outside of a request!")
-        end
-      end
-    end
-
-    # Checks whether a given line hash is a header line.
-    def header_line?(hash)
-      file_format.line_definitions[hash[:line_type]].header
-    end
-
-    # Checks whether a given line hash is a footer line.    
-    def footer_line?(hash)
-      file_format.line_definitions[hash[:line_type]].footer  
-    end 
-  end
-
-end

data/lib/request_log_analyzer/tracker/base.rb

@@ -1,54 +0,0 @@
-module RequestLogAnalyzer
-  module Tracker
-   
-    # Base tracker. All other trackers inherit from this class
-    #
-    # Accepts the following options:
-    # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
-    # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
-    # * <tt>:output</tt> Direct output here (defaults to STDOUT)
-    #
-    # For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
-    class Base
-
-      attr_reader :options
-      
-      def initialize(options ={})
-        @options = options
-      end
-      
-      def prepare
-      end
-      
-      def update(request)
-      end
-      
-      def finalize
-      end
-      
-      def should_update?(request)
-        return false if options[:line_type] && !request.has_line_type?(options[:line_type])
-        
-        if options[:if].kind_of?(Symbol)
-          return false unless request[options[:if]]
-        elsif options[:if].respond_to?(:call)
-          return false unless options[:if].call(request)
-        end
-        
-        if options[:unless].kind_of?(Symbol)
-          return false if request[options[:unless]]
-        elsif options[:unless].respond_to?(:call)
-          return false if options[:unless].call(request)
-        end
-        
-        return true
-      end
-      
-      def report(output=STDOUT, report_width = 80, color = false)
-        output << self.inspect
-        output << "\n"  
-      end
-           
-    end 
-  end
-end

data/lib/request_log_analyzer/tracker/category.rb

@@ -1,71 +0,0 @@
-module RequestLogAnalyzer::Tracker
-
-  # Catagorize requests.
-  # Count and analyze requests for a specific attribute 
-  #
-  # Accepts the following options:
-  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
-  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
-  # * <tt>:title</tt> Title do be displayed above the report.
-  # * <tt>:category</tt> Proc that handles the request categorization.
-  # * <tt>:amount</tt> The amount of lines in the report
-  #
-  # The items in the update request hash are set during the creation of the Duration tracker.
-  #
-  # Example output:
-  #  HTTP methods
-  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-  #  GET    ┃  22248 hits (46.2%) ┃░░░░░░░░░░░░░░░░░
-  #  PUT    ┃  13685 hits (28.4%) ┃░░░░░░░░░░░
-  #  POST   ┃  11662 hits (24.2%) ┃░░░░░░░░░
-  #  DELETE ┃    512 hits (1.1%)  ┃
-  class Category < RequestLogAnalyzer::Tracker::Base
-  
-    attr_reader :categories
-  
-    def prepare
-      raise "No categorizer set up for category tracker #{self.inspect}" unless options[:category]
-      @categories = {}
-      if options[:all_categories].kind_of?(Enumerable)
-        options[:all_categories].each { |cat| @categories[cat] = 0 }
-      end
-    end
-              
-    def update(request)
-      cat = options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
-      if !cat.nil? || options[:nils]
-        @categories[cat] ||= 0
-        @categories[cat] += 1
-      end
-    end
-  
-    def report(output = STDOUT, report_width = 80, color = false)
-      if options[:title]
-        output << "\n#{options[:title]}\n"
-        output << green(('━' * report_width), color) + "\n"
-      end
-    
-      if @categories.empty?
-        output << "None found.\n" 
-      else
-        sorted_categories = @categories.sort { |a, b| b[1] <=> a[1] }
-        total_hits        = sorted_categories.inject(0) { |carry, item| carry + item[1] }
-        sorted_categories = sorted_categories.slice(0...options[:amount]) if options[:amount]
-
-        adjuster = color ? 33 : 24 # justifcation calcultaion is slight different when color codes are inserterted
-        max_cat_length = [sorted_categories.map { |c| c[0].to_s.length }.max, report_width - adjuster].min
-        sorted_categories.each do |(cat, count)|
-          text = "%-#{max_cat_length+1}s┃%7d hits %s" % [cat.to_s[0..max_cat_length], count, (green("(%0.01f%%)", color) % [(count.to_f / total_hits) * 100])]
-          space_left  = report_width - (max_cat_length + adjuster + 3)
-          if space_left > 3
-            bar_chars  = (space_left * (count.to_f / total_hits)).round
-            output << "%-#{max_cat_length + adjuster}s %s%s" % [text, '┃', '░' * bar_chars] + "\n"
-          else
-            output << text + "\n"
-          end
-        end
-      end
-    end
-
-  end
-end

data/spec/controller_spec.rb

@@ -1,40 +0,0 @@
-require File.dirname(__FILE__) + '/spec_helper'
-
-describe RequestLogAnalyzer::Controller do
-
-  include RequestLogAnalyzerSpecHelper
-
-  # it "should include the file format module" do
-  #   controller = RequestLogAnalyzer::Controller.new(:rails)
-  #   (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
-  # end
-
-  it "should call the aggregators when run" do
-    file_format = RequestLogAnalyzer::FileFormat.load(:rails)
-    source      = RequestLogAnalyzer::Source::LogFile.new(file_format, :source_files => log_fixture(:rails_1x))  
-    controller  = RequestLogAnalyzer::Controller.new(source)
-    
-    mock_aggregator = mock('aggregator')
-    mock_aggregator.should_receive(:prepare).once.ordered
-    mock_aggregator.should_receive(:aggregate).with(an_instance_of(RequestLogAnalyzer::Request)).at_least(:twice).ordered
-    mock_aggregator.should_receive(:finalize).once.ordered
-    mock_aggregator.should_receive(:report).once.ordered
-    
-    another_mock_aggregator = mock('another aggregator')
-    another_mock_aggregator.should_receive(:prepare).once.ordered
-    another_mock_aggregator.should_receive(:aggregate).with(an_instance_of(RequestLogAnalyzer::Request)).at_least(:twice).ordered
-    another_mock_aggregator.should_receive(:finalize).once.ordered  
-    another_mock_aggregator.should_receive(:report).once.ordered  
-
-    
-    controller.aggregators << mock_aggregator << another_mock_aggregator
-    controller.run!
-  end
-  
-  it "should run well from the command line" do
-    temp_file = "#{File.dirname(__FILE__)}/fixtures/temp.txt"
-    system("#{File.dirname(__FILE__)}/../bin/request-log-analyzer #{log_fixture(:rails_1x)} > #{temp_file}").should be_true
-    File.unlink(temp_file)
-  end
-  
-end

data/spec/database_inserter_spec.rb

@@ -1,101 +0,0 @@
-require File.dirname(__FILE__) + '/spec_helper'
-require 'request_log_analyzer/aggregator/database'
-
-
-describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
-
-  TEST_DATABASE_FILE = File.dirname(__FILE__) + "/fixtures/requests.db"
-  include RequestLogAnalyzerSpecHelper
-  
-  before(:each) do
-    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
-    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
-  end
-  
-  after(:each) do
-    File.unlink(TEST_DATABASE_FILE) if File.exist?(TEST_DATABASE_FILE)
-  end
-  
-  it "should create the correct tables" do
-    ActiveRecord::Migration.should_receive(:create_table).with("warnings")    
-    ActiveRecord::Migration.should_receive(:create_table).with("requests")        
-    ActiveRecord::Migration.should_receive(:create_table).with("first_lines")
-    ActiveRecord::Migration.should_receive(:create_table).with("test_lines")        
-    ActiveRecord::Migration.should_receive(:create_table).with("last_lines")    
-    @database_inserter.prepare
-  end
-  
-  it "should create a default Request class" do
-    @database_inserter.prepare
-    SpecFormat::Database::Request.ancestors.should include(ActiveRecord::Base)
-    SpecFormat::Database::Request.column_names.should include('first_lineno')
-    SpecFormat::Database::Request.column_names.should include('last_lineno')    
-  end
-  
-  it "should create associations for the default Request class" do
-    @database_inserter.prepare
-    @request = SpecFormat::Database::Request.new
-    @request.should respond_to(:test_lines)
-    @request.test_lines.should
-  end
-  
-  it "should create the default table names" do
-    @database_inserter.prepare
-    @database_inserter.file_format.line_definitions.each do |name, definition|
-      klass = SpecFormat::Database.const_get("#{name}_line".camelize)
-      klass.column_names.should include('id')
-      klass.column_names.should include('lineno')      
-      klass.column_names.should include('request_id')      
-    end
-  end
-  
-  it "should create the correct fields in the table" do
-    @database_inserter.prepare
-    
-    SpecFormat::Database::FirstLine.column_names.should include('request_no')
-    SpecFormat::Database::LastLine.column_names.should include('request_no')
-    SpecFormat::Database::TestLine.column_names.should include('test_capture')    
-  end
-  
-end
-
-describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
-  include RequestLogAnalyzerSpecHelper  
-  
-  before(:each) do
-    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)    
-    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
-    @database_inserter.prepare
-        
-    @incomplete_request = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
-    @completed_request = RequestLogAnalyzer::Request.create(spec_format, 
-                          {:line_type => :first, :request_no  => 564},
-                          {:line_type => :test, :test_capture => "awesome"},
-                          {:line_type => :test, :test_capture => "indeed"},                                                    
-                          {:line_type => :last, :request_no   => 564})    
-  end
-  
-  after(:each) do
-    File.unlink(TEST_DATABASE_FILE) if File.exist?(TEST_DATABASE_FILE)
-  end 
-  
-  it "should insert a record in the request table" do
-    SpecFormat::Database::Request.count.should == 0
-    @database_inserter.aggregate(@incomplete_request)
-    SpecFormat::Database::Request.count.should == 1
-  end
-  
-  it "should insert records in all relevant line tables" do
-    @database_inserter.aggregate(@completed_request)
-    request = SpecFormat::Database::Request.first
-    request.should have(2).test_lines
-    request.should have(1).first_lines  
-    request.should have(1).last_lines    
-  end
-  
-  it "should log a warning in the warnings table" do
-    SpecFormat::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
-    @database_inserter.warning(:test_warning, "Testing the warning system", 12)
-  end
-  
-end