wvanbergen-request-log-analyzer 1.1.0 → 1.1.1

data/spec/controller_spec.rb

@@ -12,22 +12,23 @@ describe RequestLogAnalyzer::Controller do
   it "should call the aggregators when run" do
     
     mock_output = mock('output')
+    mock_output.stub!(:io).and_return($stdout)
     mock_output.should_receive(:header)
-    mock_output.should_receive(:footer)    
+    mock_output.should_receive(:footer)
     
     file_format = RequestLogAnalyzer::FileFormat.load(:rails)
-    source      = RequestLogAnalyzer::Source::LogFile.new(file_format, :source_files => log_fixture(:rails_1x))  
+    source      = RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => log_fixture(:rails_1x))  
     controller  = RequestLogAnalyzer::Controller.new(source, :output => mock_output)
     
     mock_aggregator = mock('aggregator')
     mock_aggregator.should_receive(:prepare).once.ordered
-    mock_aggregator.should_receive(:aggregate).with(an_instance_of(RequestLogAnalyzer::Request)).at_least(:twice).ordered
+    mock_aggregator.should_receive(:aggregate).with(an_instance_of(file_format.class::Request)).at_least(:twice).ordered
     mock_aggregator.should_receive(:finalize).once.ordered
     mock_aggregator.should_receive(:report).once.ordered
     
     another_mock_aggregator = mock('another aggregator')
     another_mock_aggregator.should_receive(:prepare).once.ordered
-    another_mock_aggregator.should_receive(:aggregate).with(an_instance_of(RequestLogAnalyzer::Request)).at_least(:twice).ordered
+    another_mock_aggregator.should_receive(:aggregate).with(an_instance_of(file_format.class::Request)).at_least(:twice).ordered
     another_mock_aggregator.should_receive(:finalize).once.ordered  
     another_mock_aggregator.should_receive(:report).once.ordered  
 

data/spec/database_inserter_spec.rb

@@ -8,7 +8,7 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
+    log_parser = RequestLogAnalyzer::Source::LogParser.new(spec_format)
     @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
   end
   
@@ -68,16 +68,13 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
   include RequestLogAnalyzerSpecHelper  
   
   before(:each) do
-    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)    
+    log_parser = RequestLogAnalyzer::Source::LogParser.new(spec_format)    
     @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
     @database_inserter.prepare
         
-    @incomplete_request = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
-    @completed_request = RequestLogAnalyzer::Request.create(spec_format, 
-                          {:line_type => :first, :request_no  => 564},
-                          {:line_type => :test, :test_capture => "awesome"},
-                          {:line_type => :test, :test_capture => "indeed"},                                                    
-                          {:line_type => :last, :request_no   => 564})    
+    @incomplete_request = spec_format.create_request( {:line_type => :first, :request_no => 564})
+    @completed_request = spec_format.create_request( {:line_type => :first, :request_no  => 564}, {:line_type => :test, :test_capture => "awesome"},
+                          {:line_type => :test, :test_capture => "indeed"}, {:line_type => :last, :request_no   => 564})    
   end
   
   after(:each) do

data/spec/file_format_spec.rb

@@ -3,8 +3,8 @@ require File.dirname(__FILE__) + '/spec_helper'
 describe RequestLogAnalyzer::FileFormat, :format_definition do
   
   before(:each) do
-    @first_file_format = Class.new(RequestLogAnalyzer::FileFormat)
-    @second_file_format = Class.new(RequestLogAnalyzer::FileFormat)
+    @first_file_format = Class.new(RequestLogAnalyzer::FileFormat::Base)
+    @second_file_format = Class.new(RequestLogAnalyzer::FileFormat::Base)
   end
     
   it "should specify lines with a hash" do

data/spec/file_formats/spec_format.rb

@@ -1,4 +1,4 @@
-class SpecFormat < RequestLogAnalyzer::FileFormat
+class SpecFormat < RequestLogAnalyzer::FileFormat::Base
 
   format_definition.first do |line|
     line.header = true
@@ -23,4 +23,5 @@ class SpecFormat < RequestLogAnalyzer::FileFormat
   report do |analyze|
     analyze.category :test_capture, :title => 'What is testing exactly?'
   end
+  
 end

data/spec/filter_spec.rb

@@ -1,7 +1,4 @@
 require File.dirname(__FILE__) + '/spec_helper'
-require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
-require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
-require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/anonimize'
 
 describe RequestLogAnalyzer::Filter::Timespan, 'both before and after'  do
   include RequestLogAnalyzerSpecHelper

data/spec/log_parser_spec.rb

@@ -1,10 +1,10 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-describe RequestLogAnalyzer::LogParser, :requests do
+describe RequestLogAnalyzer::Source::LogParser, :requests do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
+    @log_parser = RequestLogAnalyzer::Source::LogParser.new(spec_format)
   end
   
   it "should have multiple line definitions" do
@@ -16,13 +16,13 @@ describe RequestLogAnalyzer::LogParser, :requests do
   end
   
   it "should parse more lines than requests" do
-    @log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).twice
+    @log_parser.should_receive(:handle_request).with(an_instance_of(SpecFormat::Request)).twice
     @log_parser.parse_file(log_fixture(:test_language_combined))
     @log_parser.parsed_lines.should > 2    
   end
   
   it "should parse requests spanned over multiple files" do
-    @log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).once
+    @log_parser.should_receive(:handle_request).with(an_instance_of(SpecFormat::Request)).once
     @log_parser.parse_files([log_fixture(:multiple_files_1), log_fixture(:multiple_files_2)])
   end
   
@@ -47,11 +47,11 @@ describe RequestLogAnalyzer::LogParser, :requests do
   
 end
 
-describe RequestLogAnalyzer::LogParser, :warnings do
+describe RequestLogAnalyzer::Source::LogParser, :warnings do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
+    @log_parser = RequestLogAnalyzer::Source::LogParser.new(spec_format)
   end
     
   it "should warn about teaser matching problems" do

data/spec/merb_format_spec.rb

@@ -1,38 +1,38 @@
-# require File.dirname(__FILE__) + '/spec_helper'
-# 
-# describe RequestLogAnalyzer::LogParser, "Merb" do
-#   include RequestLogAnalyzerSpecHelper
-#   
-#   before(:each) do
-#     @log_parser = RequestLogAnalyzer::LogParser.new(:merb)
-#   end
-#   
-#   it "should have a valid language definitions" do
-#     @log_parser.file_format.should be_valid
-#   end
-#   
-#   it "should parse a stream and find valid requests" do
-#     File.open(log_fixture(:merb), 'r') do |io|
-#       @log_parser.parse_io(io) do |request| 
-#         request.should be_kind_of(RequestLogAnalyzer::Request)
-#       end
-#     end
-#   end
-#
-#   it "should find 11 completed requests" do
-#     @log_parser.should_receive(:handle_request).exactly(11).times
-#     @log_parser.parse_file(log_fixture(:merb))
-#   end
-#   
-#   it "should parse all details from a request correctly" do
-#     request = nil
-#     @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
-#     
-#     request.should be_completed
-#     request[:timestamp].should == DateTime.parse('Fri Aug 29 11:10:23 +0200 2008')
-#     request[:dispatch_time].should == 0.243424
-#     request[:after_filters_time].should == 6.9e-05
-#     request[:before_filters_time].should == 0.213213
-#     request[:action_time].should == 0.241652
-#   end
-# end
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe RequestLogAnalyzer::Source::LogParser, :merb do
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:merb))
+  end
+  
+  it "should have a valid language definitions" do
+    @log_parser.file_format.should be_valid
+  end
+  
+  it "should parse a stream and find valid requests" do
+    File.open(log_fixture(:merb), 'r') do |io|
+      @log_parser.parse_io(io) do |request| 
+        request.should be_kind_of(RequestLogAnalyzer::Request)
+      end
+    end
+  end
+
+  it "should find 11 completed requests" do
+    @log_parser.should_receive(:handle_request).exactly(11).times
+    @log_parser.parse_file(log_fixture(:merb))
+  end
+  
+  it "should parse all details from a request correctly" do
+    request = nil
+    @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
+    
+    request.should be_completed
+    #request[:timestamp].should == DateTime.parse('Fri Aug 29 11:10:23 +0200 2008') # FIX ME
+    request[:dispatch_time].should == 0.243424
+    request[:after_filters_time].should == 6.9e-05
+    request[:before_filters_time].should == 0.213213
+    request[:action_time].should == 0.241652
+  end
+end

data/spec/rails_format_spec.rb

@@ -1,10 +1,10 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-describe RequestLogAnalyzer::LogParser, "Rails" do
+describe RequestLogAnalyzer::Source::LogParser, "Rails" do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails))
+    @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails))
   end
   
   it "should have a valid language definitions" do

data/spec/request_spec.rb

@@ -5,7 +5,7 @@ describe RequestLogAnalyzer::Request, :incomplete_request do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @incomplete_request = RequestLogAnalyzer::Request.new(spec_format)
+    @incomplete_request = spec_format.create_request
     @incomplete_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
   end
   
@@ -37,7 +37,7 @@ describe RequestLogAnalyzer::Request, :completed_request do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @completed_request = RequestLogAnalyzer::Request.new(spec_format)
+    @completed_request = spec_format.create_request
     @completed_request << { :line_type => :first, :lineno =>  1, :name => 'first line!' }    
     @completed_request << { :line_type => :test,  :lineno =>  4, :test_capture => 'testing' }        
     @completed_request << { :line_type => :test,  :lineno =>  7, :test_capture => 'testing some more' }            

data/spec/spec_helper.rb

@@ -21,47 +21,13 @@ module RequestLogAnalyzerSpecHelper
     File.dirname(__FILE__) + "/fixtures/#{name}.log"
   end
   
-  def request(fields, format = TestFileFormat)
+  def request(fields, format = spec_format)
     if fields.kind_of?(Array)
-      RequestLogAnalyzer::Request.create(format, *fields)
+      format.create_request(*fields)
     else
-      RequestLogAnalyzer::Request.create(format, fields)
+      format.create_request(fields)      
     end
   end
   
 end
 
-module TestFileFormat
-  
-  module Summarizer
-    def self.included(base)
-      # monkey patching for summarizer here :-)
-    end
-  end
-  
-  module LogParser
-    def self.included(base)
-      # monkey patching for log parser here :-)
-    end
-  end
-  
-  LINE_DEFINITIONS = {
-    :first => {
-      :header => true,
-      :teaser => /processing /,
-      :regexp => /processing request (\d+)/,
-      :captures => [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]    
-    },
-    :test => {
-      :teaser => /testing /,
-      :regexp => /testing is (\w+)/,
-      :captures => [{ :name => :test_capture, :type => :string, :anonymize => true}]
-    }, 
-    :last => {
-      :footer => true,
-      :teaser => /finishing /,
-      :regexp => /finishing request (\d+)/,
-      :captures => [{ :name => :request_no, :type => :integer}]
-    }
-  }
-end

data/tasks/github-gem.rake

@@ -241,7 +241,8 @@ module Rake
     
     def release_task
       puts
-      puts "Released #{@name} - version#{@specification.version}"
+      puts '------------------------------------------------------------'
+      puts "Released #{@name} - version #{@specification.version}"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-18 00:00:00 -08:00
+date: 2009-01-21 00:00:00 -08:00
 default_executable: request-log-analyzer
 dependencies: []
 
@@ -37,7 +37,7 @@ files:
 - lib/request_log_analyzer
 - lib/request_log_analyzer.rb
 - lib/request_log_analyzer/aggregator
-- lib/request_log_analyzer/aggregator/base.rb
+- lib/request_log_analyzer/aggregator.rb
 - lib/request_log_analyzer/aggregator/database.rb
 - lib/request_log_analyzer/aggregator/echo.rb
 - lib/request_log_analyzer/aggregator/summarizer.rb
@@ -48,12 +48,11 @@ files:
 - lib/request_log_analyzer/file_format/rails.rb
 - lib/request_log_analyzer/file_format/rails_development.rb
 - lib/request_log_analyzer/filter
+- lib/request_log_analyzer/filter.rb
 - lib/request_log_analyzer/filter/anonimize.rb
-- lib/request_log_analyzer/filter/base.rb
 - lib/request_log_analyzer/filter/field.rb
 - lib/request_log_analyzer/filter/timespan.rb
 - lib/request_log_analyzer/line_definition.rb
-- lib/request_log_analyzer/log_parser.rb
 - lib/request_log_analyzer/log_processor.rb
 - lib/request_log_analyzer/output
 - lib/request_log_analyzer/output.rb
@@ -61,10 +60,10 @@ files:
 - lib/request_log_analyzer/output/html.rb
 - lib/request_log_analyzer/request.rb
 - lib/request_log_analyzer/source
-- lib/request_log_analyzer/source/base.rb
-- lib/request_log_analyzer/source/log_file.rb
+- lib/request_log_analyzer/source.rb
+- lib/request_log_analyzer/source/log_parser.rb
 - lib/request_log_analyzer/tracker
-- lib/request_log_analyzer/tracker/base.rb
+- lib/request_log_analyzer/tracker.rb
 - lib/request_log_analyzer/tracker/category.rb
 - lib/request_log_analyzer/tracker/duration.rb
 - lib/request_log_analyzer/tracker/hourly_spread.rb

data/lib/request_log_analyzer/filter/base.rb

@@ -1,32 +0,0 @@
-module RequestLogAnalyzer
-  module Filter
-    # Base filter class used to filter input requests.
-    # All filters should interit from this base.
-    class Base
-      
-      include RequestLogAnalyzer::FileFormat::Awareness
-      
-      attr_reader :log_parser
-      attr_reader :options
-      
-      # Initializer
-      # <tt>format</tt> The file format
-      # <tt>options</tt> Are passed to the filters.
-      def initialize(format, options = {})
-        @options    = options
-        register_file_format(format)
-      end
-      
-      # Initialize the filter
-      def prepare
-      end
-      
-      # Return the request if the request should be kept.
-      # Return nil otherwise.
-      def filter(request)
-        return nil unless request
-        return request
-      end
-    end
-  end
-end

data/lib/request_log_analyzer/log_parser.rb

@@ -1,173 +0,0 @@
-module RequestLogAnalyzer
-  
-  # The LogParser class reads log data from a given source and uses a file format definition
-  # to parse all relevent information about requests from the file. A FileFormat module should 
-  # be provided that contains the definitions of the lines that occur in the log data.
-  #
-  # De order in which lines occur is used to combine lines to a single request. If these lines 
-  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
-  # the log file simultaneously by different mongrel processes. This problem is detected by the 
-  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
-  # occurs.
-  class LogParser
-    
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    # A hash of options
-    attr_reader :options
-    
-    # The current Request object that is being parsed
-    attr_reader :current_request
-    
-    # The total number of parsed lines
-    attr_reader :parsed_lines
-    
-    # The total number of parsed requests.
-    attr_reader :parsed_requests
-    
-    # The number of skipped requests because of date constraints
-    attr_reader :skipped_requests
-    
-    # Initializes the parser instance.
-    # It will apply the language specific FileFormat module to this instance. It will use the line
-    # definitions in this module to parse any input.
-    def initialize(format, options = {})      
-      @line_definitions = {}
-      @options          = options
-      @parsed_lines     = 0
-      @parsed_requests  = 0
-      @skipped_requests = 0      
-      
-      @current_io = nil
-      
-      # install the file format module (see RequestLogAnalyzer::FileFormat)
-      # and register all the line definitions to the parser
-      self.register_file_format(format)
-    end
-    
-    # Parses a list of consequent files of the same format
-    def parse_files(files, options = {}, &block)
-      files.each { |file| parse_file(file, options, &block) }
-    end
-    
-    # Parses a file. 
-    # Creates an IO stream for the provided file, and sends it to parse_io for further handling
-    def parse_file(file, options = {}, &block)
-      @progress_handler.call(:started, file) if @progress_handler
-      File.open(file, 'r') { |f| parse_io(f, options, &block) }
-      @progress_handler.call(:finished, file) if @progress_handler
-    end
-    
-    def parse_stream(stream, options = {}, &block)
-      parse_io(stream, options, &block)
-    end
-
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def parse_io(io, options = {}, &block)
-
-      # parse every line type by default
-      line_types = options[:line_types] || file_format.line_definitions.keys
-
-      # check whether all provided line types are valid
-      unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
-      raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
-      
-      @current_io = io
-      @current_io.each_line do |line|
-        
-        @progress_handler.call(:progress, @current_io.pos) if @progress_handler && @current_io.kind_of?(File)
-        
-        request_data = nil
-        line_types.each do |line_type|
-          line_type_definition = file_format.line_definitions[line_type]
-          break if request_data = line_type_definition.matches(line, @current_io.lineno, self)
-        end
-
-        if request_data
-          @parsed_lines += 1
-          update_current_request(request_data, &block)
-        end
-      end
-      
-      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
-  
-      @current_io = nil
-    end
-    
-    # Add a block to this method to install a progress handler while parsing
-    def progress=(proc)
-      @progress_handler = proc
-    end
-    
-    # Add a block to this method to install a warning handler while parsing
-    def warning=(proc)
-      @warning_handler = proc
-    end
-    
-    # This method is called by the parser if it encounteres any problems.
-    # It will call the warning handler. The default controller will pass all warnings to every
-    # aggregator that is registered and running
-    def warn(type, message)
-      @warning_handler.call(type, message, @current_io.lineno) if @warning_handler
-    end
-    
-    protected
-    
-    # Combines the different lines of a request into a single Request object. It will start a 
-    # new request when a header line is encountered en will emit the request when a footer line 
-    # is encountered.
-    #
-    # - Every line that is parsed before a header line is ignored as it cannot be included in 
-    #   any request. It will emit a :no_current_request warning.
-    # - A header line that is parsed before a request is closed by a footer line, is a sign of
-    #   an unprpertly ordered file. All data that is gathered for the request until then is 
-    #   discarded, the next request is ignored as well and a :unclosed_request warning is
-    #   emitted.
-    def update_current_request(request_data, &block)
-      if header_line?(request_data)
-        unless @current_request.nil?
-          if options[:assume_correct_order]
-            handle_request(@current_request, &block)            
-            @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
-          else
-            warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
-            @current_request = nil # remove all data that was parsed, skip next request as well.
-          end
-        else
-          @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)              
-        end
-      else
-        unless @current_request.nil?
-          @current_request << request_data
-          if footer_line?(request_data)
-            handle_request(@current_request, &block)
-            @current_request = nil 
-          end
-        else
-          warn(:no_current_request, "Parsebale line found outside of a request!")
-        end
-      end
-    end
-    
-    # Handles the parsed request by calling the request handler.
-    # The default controller will send the request to every running aggegator.
-    def handle_request(request, &block)
-      @parsed_requests += 1
-      accepted = block_given? ? yield(request) : true
-      @skipped_requests += 1 if !accepted
-    end
-    
-    # Checks whether a given line hash is a header line.
-    def header_line?(hash)
-      file_format.line_definitions[hash[:line_type]].header
-    end
-
-    # Checks whether a given line hash is a footer line.    
-    def footer_line?(hash)
-      file_format.line_definitions[hash[:line_type]].footer  
-    end 
-  end
-end

data/lib/request_log_analyzer/tracker/base.rb

@@ -1,54 +0,0 @@
-module RequestLogAnalyzer
-  module Tracker
-   
-    # Base tracker. All other trackers inherit from this class
-    #
-    # Accepts the following options:
-    # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
-    # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
-    # * <tt>:output</tt> Direct output here (defaults to STDOUT)
-    #
-    # For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
-    class Base
-
-      attr_reader :options
-      
-      def initialize(options ={})
-        @options = options
-      end
-      
-      def prepare
-      end
-      
-      def update(request)
-      end
-      
-      def finalize
-      end
-      
-      def should_update?(request)
-        return false if options[:line_type] && !request.has_line_type?(options[:line_type])
-        
-        if options[:if].kind_of?(Symbol)
-          return false unless request[options[:if]]
-        elsif options[:if].respond_to?(:call)
-          return false unless options[:if].call(request)
-        end
-        
-        if options[:unless].kind_of?(Symbol)
-          return false if request[options[:unless]]
-        elsif options[:unless].respond_to?(:call)
-          return false if options[:unless].call(request)
-        end
-        
-        return true
-      end
-      
-      def report(output)
-        output << self.inspect
-        output << "\n"  
-      end
-           
-    end 
-  end
-end