ngmoco-request-log-analyzer 1.4.2

data/spec/unit/file_format/line_definition_spec.rb

@@ -0,0 +1,75 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::LineDefinition do
+
+  before(:each) do
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, {
+      :teaser   => /Testing /,
+      :regexp   => /Testing (\w+), tries\: (\d+)/,
+      :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer }]
+    })
+  end
+
+  describe '#matches' do
+
+    it "should return false on an unmatching line" do
+      @line_definition.matches("nonmatching").should be_false
+    end
+
+    it "should return false when only the teaser matches" do
+      @line_definition.matches("Testing LineDefinition").should be_false
+    end
+
+    it "should parse a line and capture the expected values" do
+      @line_definition.matches("Testing LineDefinition, tries: 123").should == {:line_definition => @line_definition, :captures => ['LineDefinition', '123'] }
+    end
+
+    it "should know which names it can capture" do
+      @line_definition.captures?(:what).should be_true
+      @line_definition.captures?(:tries).should be_true
+      @line_definition.captures?(:bogus).should be_false
+    end
+    
+  end
+
+  describe '#convert_captured_values' do
+
+    before(:each) do
+      @request = mock('request')
+      @request.stub!(:convert_value).and_return('foo')
+    end
+
+    it "should call convert_value for every captured value" do
+      @request.should_receive(:convert_value).twice
+      @line_definition.convert_captured_values(['test', '123'], @request)
+    end
+
+    it "should set the converted values" do
+      @line_definition.convert_captured_values(['test', '123'], @request).should == {:what => 'foo', :tries => 'foo'}
+    end
+
+    context 'when using :provides option' do
+      before(:each) do
+        @ld = RequestLogAnalyzer::LineDefinition.new(:test, :regexp   => /Hash\: (\{.+\})/,
+            :captures => [{ :name => :hash, :type => :hash, :provides => {:bar => :string}}])
+
+        @request = mock('request')
+
+        @request.stub!(:convert_value).with("{:bar=>'baz'}", anything).and_return(:bar => 'baz')
+        @request.stub!(:convert_value).with('baz', anything).and_return('foo')
+      end
+
+      it "should call Request#convert_value for the initial hash and the value in the hash" do
+        @request.should_receive(:convert_value).with("{:bar=>'baz'}", anything).and_return(:bar => 'baz')
+        @request.should_receive(:convert_value).with("baz", anything)
+        @ld.convert_captured_values(["{:bar=>'baz'}"], @request)
+      end
+
+      it "should set the provides fields" do
+        # The captures field must be set and converted as well
+        @ld.convert_captured_values(["{:bar=>'baz'}"], @request)[:bar].should eql('foo')
+      end
+    end
+
+  end
+end

data/spec/unit/file_format/merb_format_spec.rb

@@ -0,0 +1,52 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Merb do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:merb).should be_valid
+  end
+
+  describe '#parse_line' do
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:merb)
+    end
+    
+    it "should parse a :started line correctly" do
+      line = '~ Started request handling: Fri Aug 29 11:10:23 +0200 2008'
+      @file_format.should parse_line(line).as(:started).and_capture(:timestamp => 20080829111023)
+    end
+
+    it "should parse a prefixed :started line correctly" do
+      line = '~ Aug 31 18:35:24 typekit-web001 merb:  ~ Started request handling: Mon Aug 31 18:35:25 +0000 2009'
+      @file_format.should parse_line(line).as(:started).and_capture(:timestamp => 20090831183525)
+    end
+
+    it "should parse a :params line correctly" do
+      line = '~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"delete", "controller"=>"session"}'
+      @file_format.should parse_line(line).as(:params).and_capture(:controller => 'session', :action => 'delete', :namespace => nil)
+    end
+
+    it "should parse a :completed line correctly" do
+      line = '~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}'
+      @file_format.should parse_line(line).as(:completed).and_capture(:dispatch_time => 0.006117, 
+        :before_filters_time => 0.000712, :action_time => 0.005833, :after_filters_time => 6.1e-05)
+    end
+  end
+
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:merb))
+    end
+
+    it "should parse a stream and find valid Merb requests" do
+      @log_parser.parse_file(log_fixture(:merb)) do |request|
+        request.should be_kind_of(RequestLogAnalyzer::FileFormat::Merb::Request)
+      end
+    end
+  
+    it "should find 11 completed requests" do
+      @log_parser.should_receive(:handle_request).exactly(11).times
+      @log_parser.parse_file(log_fixture(:merb))
+    end
+  end
+end

data/spec/unit/file_format/rails_format_spec.rb

@@ -0,0 +1,164 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Rails do
+
+  describe '.create' do
+
+    context 'without providing a lines argument' do
+      before(:each) { @rails = RequestLogAnalyzer::FileFormat.load(:rails) }
+
+      it "should create a valid file format" do
+        @rails.should be_valid
+      end
+
+      it "should parse the production lines" do
+        production_rails = RequestLogAnalyzer::FileFormat.load(:rails, 'production')
+        @rails.line_definitions.should == production_rails.line_definitions
+      end
+    end
+
+    context 'using a comma separated list of lines as argument' do
+      before(:each) { @rails = RequestLogAnalyzer::FileFormat.load(:rails, 'minimal,failure') }
+
+      it "should return a valid language" do
+        @rails.should be_valid
+      end
+      
+      it "should at least parse :processing and :completed lines" do
+        @rails.line_definitions.should include(:processing, :completed, :failure)
+      end
+    end
+
+    RequestLogAnalyzer::FileFormat::Rails::LINE_COLLECTIONS.keys.each do |constant|
+
+      context "using the '#{constant}' line collection constant" do
+        before(:each) { @rails = RequestLogAnalyzer::FileFormat.load(:rails, constant) }
+
+        it "should return a valid language" do
+          @rails.should be_valid
+        end
+
+        it "should at least parse :processing and :completed lines" do
+          @rails.line_definitions.should include(:processing, :completed)
+        end
+      end
+    end
+  end
+
+  describe '#parse_line' do
+    before(:each) { @rails = RequestLogAnalyzer::FileFormat.load(:rails, :all) }
+
+    {'with prefix' => 'LINE PREFIX: ', 'without prefix' => '' }.each do |context, prefix|
+      context context do
+        it "should parse a :processing line correctly" do
+          line = prefix + 'Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]'
+          @rails.should parse_line(line).as(:processing).and_capture(:controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET')
+        end
+
+        it "should parse a Rails 2.1 style :completed line correctly" do
+          line = prefix + 'Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]'
+          @rails.should parse_line(line).as(:completed).and_capture(:duration => 0.21665, :db => 0.0, :view => 0.00926, :status => 200, :url => 'http://demo.nu/employees')
+        end
+
+        it "should parse a Rails 2.2 style :completed line correctly" do
+          line = prefix + 'Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]'
+          @rails.should parse_line(line).as(:completed).and_capture(:duration => 0.614, :db => 0.031, :view => 0.120, :status => 200, :url => 'http://floorplanner.local/demo')
+        end
+
+        it "should parse a :failure line with exception correctly" do
+          line = prefix + "NoMethodError (undefined method `update_domain_account' for nil:NilClass):"
+          @rails.should parse_line(line).as(:failure).and_capture(:error => 'NoMethodError', :message => "undefined method `update_domain_account' for nil:NilClass")
+        end
+
+        it "should parse a :cache_hit line correctly" do
+          line = prefix + 'Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.'
+          @rails.should parse_line(line).as(:cache_hit)
+        end
+
+        it "should parse a :parameters line correctly" do
+          line = prefix + '  Parameters: {"action"=>"cached", "controller"=>"cached"}'
+          @rails.should parse_line(line).as(:parameters).and_capture(:params => {:action => 'cached', :controller => 'cached'})
+        end
+
+        it "should parse a :rendered line correctly" do
+          line = prefix + 'Rendered layouts/_footer (2.9ms)'
+          @rails.should parse_line(line).as(:rendered).and_capture(:render_file => 'layouts/_footer', :render_duration => 0.0029)
+        end
+
+        it "should parse a :query_executed line with colors" do
+          line = prefix + ' User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) '
+          @rails.should parse_line(line).as(:query_executed).and_capture(:query_class => 'User', :query_duration => 0.0004, :query_sql => 'SELECT * FROM users WHERE (users.id = :int)')
+        end
+
+        it "should parse a :query_executed line without colors" do
+          line = prefix + ' User Load (0.4ms)   SELECT * FROM `users` WHERE (`users`.`id` = 18205844) '
+          @rails.should parse_line(line).as(:query_executed).and_capture(:query_class => 'User', :query_duration => 0.0004, :query_sql => 'SELECT * FROM users WHERE (users.id = :int)')
+        end
+
+        it "should parse a :query_cached line with colors" do
+          line = prefix + ' CACHE (0.0ms)   SELECT * FROM `users` WHERE (`users`.`id` = 0) '
+          @rails.should parse_line(line).as(:query_cached).and_capture(:cached_duration => 0.0, :cached_sql => 'SELECT * FROM users WHERE (users.id = :int)')
+        end
+
+        it "should parse a :query_cached line without colors" do
+          line = prefix + ' CACHE (0.0ms)   SELECT * FROM `users` WHERE (`users`.`id` = 0) '
+          @rails.should parse_line(line).as(:query_cached).and_capture(:cached_duration => 0.0, :cached_sql => 'SELECT * FROM users WHERE (users.id = :int)')
+        end
+
+        it "should not parse an unsupported line" do
+          line = prefix + 'nonsense line that should not be parsed as anything'
+          @rails.should_not parse_line(line)
+        end
+      end
+    end
+  end
+
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(
+            RequestLogAnalyzer::FileFormat.load(:rails), :parse_strategy => 'cautious')
+    end
+
+    it "should parse a Rails 2.1 style log and find valid Rails requests without warnings" do
+      request_counter.should_receive(:hit!).exactly(4).times
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_file(log_fixture(:rails_1x)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
+      end
+    end
+
+    it "should parse a Rails 2.2 style log and find valid Rails requests without warnings" do
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_file(log_fixture(:rails_22)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
+      end
+    end
+
+    it "should parse a Rails SyslogLogger file with prefix and find valid requests without warnings" do
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_file(log_fixture(:syslog_1x)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
+      end
+    end
+
+    it "should parse cached requests" do
+      @log_parser.should_not_receive(:warn)
+      @log_parser.parse_file(log_fixture(:rails_22_cached)) do |request|
+        request.should be_completed
+        request =~ :cache_hit
+      end
+    end
+
+    it "should detect unordered requests in the logs" do
+      @log_parser.should_not_receive(:handle_request)
+      @log_parser.should_receive(:warn).with(:unclosed_request, anything).once
+      @log_parser.should_receive(:warn).with(:no_current_request, anything).twice
+      @log_parser.parse_file(log_fixture(:rails_unordered))
+    end
+  end
+end

data/spec/unit/filter/anonymize_filter_spec.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Anonymize.new(testing_format)
+  end
+
+  it "should anonimize ip" do
+    @filter.filter(request(:ip => '123.123.123.123'))[:ip].should_not eql('123.123.123.123')
+  end
+
+  it "should anonimize url" do
+    @filter.filter(request(:url => 'https://test.mysite.com/employees'))[:url].should eql('http://example.com/employees')
+  end
+
+  it "should fuzz durations" do
+    @filter.filter(request(:duration => 100))[:duration].should_not eql(100)
+  end
+
+end

data/spec/unit/filter/field_filter_spec.rb

@@ -0,0 +1,66 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => 'test', :mode => :select)
+  end
+
+  it "should reject a request if the field value does not match" do
+    @filter.filter(request(:test => 'not test')).should be_nil
+  end
+
+  it "should reject a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should be_nil
+  end
+
+  it "should accept a request if the both name and value match" do
+    @filter.filter(request(:test => 'test')).should_not be_nil
+  end
+
+  it "should accept a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
+  end
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => 'test', :mode => :reject)
+  end
+
+  it "should accept a request if the field value does not match" do
+    @filter.filter(request(:test => 'not test')).should_not be_nil
+  end
+
+  it "should accept a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should_not be_nil
+  end
+
+  it "should reject a request if the both name and value match" do
+    @filter.filter(request(:test => 'test')).should be_nil
+  end
+
+  it "should reject a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
+  end
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, :field => :test, :value => '/test/', :mode => :select)
+  end
+
+  it "should reject a request if the field value does not match" do
+    @filter.filter(request(:test => 'a working test')).should_not be_nil
+  end
+
+  it "should reject a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should be_nil
+  end
+
+  it "should accept a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
+  end
+end

data/spec/unit/filter/filter_spec.rb

@@ -0,0 +1,17 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Filter::Base, 'base filter' do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Base.new(testing_format)
+  end
+
+  it "should return everything" do
+    @filter.filter(request(:ip => '123.123.123.123'))[:ip].should eql('123.123.123.123')
+  end
+
+  it "should return nil on nil request" do
+    @filter.filter(nil).should be_nil
+  end
+
+end

data/spec/unit/filter/timespan_filter_spec.rb

@@ -0,0 +1,58 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Filter::Timespan, 'both before and after'  do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
+  end
+
+  it "should reject a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should be_nil
+  end
+
+  it "should reject a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should be_nil
+  end
+
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end
+end
+
+describe RequestLogAnalyzer::Filter::Timespan, 'only before'  do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :before => DateTime.parse('2009-02-02'))
+  end
+
+  it "should accept a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
+  end
+
+  it "should reject a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should be_nil
+  end
+
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end
+end
+
+describe RequestLogAnalyzer::Filter::Timespan, 'only after'  do
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, :after => DateTime.parse('2009-01-01'))
+  end
+
+  it "should reject a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should be_nil
+  end
+
+  it "should accept a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
+  end
+
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end
+end