ngmoco-request-log-analyzer 1.4.2

data/spec/unit/database/connection_spec.rb

@@ -0,0 +1,34 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Database::Connection do
+  describe '.from_string' do
+
+    it "should parse a name-value based string" do
+      string = 'adapter=sqlite3;database=filename.db'
+      RequestLogAnalyzer::Database::Connection.from_string(string).should == {:adapter => 'sqlite3', :database => 'filename.db'}
+    end
+
+    it "should parse an URI-based string for SQLite3" do
+      string = 'sqlite3://filename.db'
+      RequestLogAnalyzer::Database::Connection.from_string(string).should == {:adapter => 'sqlite3', :database => 'filename.db'}
+    end
+
+    it "should parse an URI-based string for MySQL" do
+      string = 'mysql://localhost.local/database'
+      RequestLogAnalyzer::Database::Connection.from_string(string).should ==
+              { :adapter => 'mysql', :database => 'database', :host => 'localhost.local' }
+    end
+
+    it "should parse an URI-based string for MySQL with only username" do
+      string = 'mysql://username@localhost.local/database'
+      RequestLogAnalyzer::Database::Connection.from_string(string).should ==
+              { :adapter => 'mysql', :database => 'database', :host => 'localhost.local', :username => 'username' }
+    end
+
+    it "should parse an URI-based string for MySQL with username and password" do
+      string = 'mysql://username:password@localhost.local/database'
+      RequestLogAnalyzer::Database::Connection.from_string(string).should ==
+              { :adapter => 'mysql', :database => 'database', :host => 'localhost.local', :username => 'username', :password => 'password' }
+    end
+  end
+end

data/spec/unit/database/database_spec.rb

@@ -0,0 +1,133 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Database do
+
+  describe '#load_database_schema!' do
+
+    context 'for a Rails request database' do
+      before(:each) do
+        @database = RequestLogAnalyzer::Database.new(log_fixture(:rails, :db))
+        @database.load_database_schema!
+      end
+
+      after(:each) { @database.remove_orm_classes! }
+
+      # FileFormat-agnostic classes
+      default_orm_class_names.each do |const|
+        it "should create the default #{const} constant" do
+          RequestLogAnalyzer::Database.const_defined?(const).should be_true
+        end
+
+        it "should create the default #{const} class inheriting from ActiveRecord::Base and RequestLogAnalyzer::Database::Base" do
+          RequestLogAnalyzer::Database.const_get(const).ancestors.should include(ActiveRecord::Base, RequestLogAnalyzer::Database::Base)
+        end
+      end
+
+      # Some Fileformat-specific classes
+      ['CompletedLine', 'ProcessingLine'].each do |const|
+        it "should create the #{const} constant" do
+          Object.const_defined?(const).should be_true
+        end
+
+        it "should create the #{const} class inheriting from ActiveRecord::Base and RequestLogAnalyzer::Database::Base" do
+          Object.const_get(const).ancestors.should include(ActiveRecord::Base, RequestLogAnalyzer::Database::Base)
+        end
+
+        it "should create a :belongs_to relation from the #{const} class to Request and Source" do
+          Object.const_get(const).send(:reflections).should include(:request, :source)
+        end
+
+        it "should create a :has_many relation from the Request and Source class to the #{const} class" do
+          RequestLogAnalyzer::Database::Request.send(:reflections).should include(const.underscore.pluralize.to_sym)
+          RequestLogAnalyzer::Database::Source.send(:reflections).should include(const.underscore.pluralize.to_sym)
+        end
+      end
+    end
+  end
+
+  describe '#create_database_schema!' do
+
+    before(:each) do
+      @database = RequestLogAnalyzer::Database.new
+      @database.file_format = testing_format
+      @database.stub!(:connection).and_return(mock_connection)
+
+      # Stub the expected method calls for the preparation, these will be tested separately
+      @mock_class = Class.new(RequestLogAnalyzer::Database::Base)
+      @mock_class.stub!(:create_table!)
+    end
+
+    after(:each) { @database.remove_orm_classes! }
+
+    default_orm_class_names.each do |klass|
+
+      it "should create a table for the default #{klass} class" do
+        @database.connection.should_receive(:create_table).with(klass.underscore.pluralize.to_sym)
+        @database.send :create_database_schema!
+      end
+
+      it "should create a #{klass} class inheriting from ActiveRecord and the base class of the ORM module" do
+        @database.send :create_database_schema!
+        RequestLogAnalyzer::Database.const_get(klass).ancestors.should include(ActiveRecord::Base, RequestLogAnalyzer::Database::Base)
+      end
+    end
+
+    testing_format.line_definitions.each do |name, definition|
+
+      it "should create the #{(name.to_s + '_line').camelize} class for #{name.inspect} lines" do
+        @database.send :create_database_schema!
+        Object.const_defined?("#{name}_line".camelize).should be_true
+      end
+
+      it "should create the #{name.to_s + '_lines'} table for the parsed #{name.inspect} lines" do
+        @database.connection.should_receive(:create_table).with("#{name}_lines".to_sym)
+        @database.send :create_database_schema!
+      end
+    end
+  end
+
+  describe '#load_activerecord_class' do
+
+    before(:each) do
+      @database = RequestLogAnalyzer::Database.new
+      @connection = mock_connection
+      @database.stub!(:connection).and_return(@connection)
+
+      # Mock the has_many method of the defaukt ORM classes
+      RequestLogAnalyzer::Database::Request.stub!(:has_many)
+      RequestLogAnalyzer::Database::Source.stub!(:has_many)
+
+      @mock_class = Class.new(RequestLogAnalyzer::Database::Base)
+
+      RequestLogAnalyzer::Database::Base.stub!(:subclass_from_table).and_return(@mock_class)
+      RequestLogAnalyzer::Database::Base.stub!(:subclass_from_line_definition).and_return(@mock_class)
+    end
+
+    after(:each) { @database.remove_orm_classes! }
+
+    it "should call :subclass_from_table when a table name is given as string" do
+      RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_table).and_return(@mock_class)
+      @database.load_activerecord_class('test_lines')
+    end
+
+    it "should call :subclass_from_table when a table name is given as symbol" do
+      RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_table).and_return(@mock_class)
+      @database.load_activerecord_class(:test_lines)
+    end
+
+    it "should call :subclass_from_table when a LineDefinition is given" do
+      RequestLogAnalyzer::Database::Base.should_receive(:subclass_from_line_definition).and_return(@mock_class)
+      @database.load_activerecord_class(RequestLogAnalyzer::LineDefinition.new(:test))
+    end
+
+    it "should define the class in the ORM module" do
+      @database.load_activerecord_class(:test_lines)
+      Object.const_defined?('TestLine').should be_true
+    end
+
+    it "should add the class to the line_classes array of the database" do
+      @database.load_activerecord_class(:test_lines)
+      @database.line_classes.should include(TestLine)
+    end
+  end
+end

data/spec/unit/file_format/amazon_s3_format_spec.rb

@@ -0,0 +1,49 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::AmazonS3 do
+
+  before(:each) do
+    @file_format = RequestLogAnalyzer::FileFormat.load(:amazon_s3)
+    @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+    @sample = '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1 sample-bucket [06/Oct/2006:01:42:14 +0000] 207.171.172.6 65a011a29cdf8ec533ec3d1ccaae921c C980091AD89C936A REST.GET.OBJECT object.png "GET /sample-bucket/object.png HTTP/1.1" 200 - 1243 1243 988 987 "-" "aranhabot"'
+  end
+
+  it "should be a valid file format" do
+    @file_format.should be_valid
+  end
+
+  it "should parse access lines and capture all of its fields" do
+    @file_format.should have_line_definition(:access).capturing(:bucket_owner, :bucket, :timestamp, :remote_ip, :requester,
+          :key, :operation, :total_time, :turnaround_time, :bytes_sent, :object_size, :referer, :user_agent)
+  end
+
+  it "should match the sample line" do
+    @file_format.parse_line(@sample).should include(:line_definition, :captures)
+  end
+
+  it "should not match a nonsense line" do
+    @file_format.parse_line('dsadasdas dsaadsads dsaadsads').should be_nil
+  end
+
+  it "should parse and convert the sample fields correctly" do
+    @log_parser.parse_io(StringIO.new(@sample)) do |request|
+      request[:bucket_owner].should    == '2f88111968424e6306bf4d292c0188ccb94ff9374ea2836b50a1a79f7cd656e1'
+      request[:bucket].should          == 'sample-bucket'
+      request[:remote_ip].should       == '207.171.172.6'
+      request[:key].should             == 'object.png'
+      request[:operation].should       == 'REST.GET.OBJECT'
+      request[:requester].should       == '65a011a29cdf8ec533ec3d1ccaae921c'
+      request[:request_id].should      == 'C980091AD89C936A'
+      request[:request_uri].should     == 'GET /sample-bucket/object.png HTTP/1.1'
+      request[:error_code].should      == nil
+      request[:http_status].should     == 200
+      request[:total_time].should      == 0.988
+      request[:turnaround_time].should == 0.987
+      request[:bytes_sent].should      == 1243
+      request[:object_size].should     == 1243
+      request[:user_agent].should      == 'aranhabot'
+      request[:referer].should         == nil
+    end
+  end
+
+end

data/spec/unit/file_format/apache_format_spec.rb

@@ -0,0 +1,203 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Apache do
+
+  describe '.access_line_definition' do
+    before(:each) do
+      @format_string   = '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T'
+      @line_definition = RequestLogAnalyzer::FileFormat::Apache.access_line_definition(@format_string)
+    end
+
+    it "should create a Regexp to match the line" do
+      @line_definition.regexp.should be_kind_of(Regexp)
+    end
+
+    it "should create a list of captures for the values in the lines" do
+      @line_definition.captures.should have(12).items
+    end
+
+    it "should make it a header line" do
+      @line_definition.should be_header
+    end
+
+    it "should make it a footer line" do
+      @line_definition.should be_footer
+    end
+
+    it "should capture :duration" do
+      @line_definition.captures?(:duration).should be_true
+    end
+  end
+
+  describe '.create' do
+
+    before(:each) do
+      @format_string = '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
+      @format = RequestLogAnalyzer::FileFormat::Apache.create(@format_string)
+    end
+
+    it "should create the :access line definition" do
+      @format.should have_line_definition(:access).capturing(:timestamp, :remote_host, :bytes_sent, :http_method, :path, :http_version, :http_status)
+    end
+
+    it "should be a valid file format" do
+      @format.should be_valid
+    end
+
+    it "should setup report trackers" do
+      @format.report_trackers.should_not be_empty
+    end
+  end
+
+  context '"Common" access log parsing' do
+    before(:all) do
+      @file_format     = RequestLogAnalyzer::FileFormat.load(:apache, :common)
+      @log_parser      = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+      @sample_1        = '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223'
+      @sample_2        = '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414'
+      @nonsense_sample = 'addasdsasadadssadasd'
+    end
+
+    it "should have a valid language definitions" do
+      @file_format.should be_valid
+    end
+
+    it "should not parse a valid access log line" do
+      @file_format.should_not parse_line(@nonsense_sample)
+    end
+
+    it "should read the correct values from a valid HTTP/1.0 access log line" do
+      @file_format.should parse_line(@sample_1).as(:access).and_capture(
+        :remote_host    => '1.129.119.13',
+        :remote_logname => nil,
+        :user           => nil,
+        :timestamp      => 20090908075409,
+        :http_status    => 200,
+        :http_method    => 'GET',
+        :http_version   => '1.0',
+        :bytes_sent     => 8223)
+    end
+
+    it "should read the correct values from a valid 200 access log line" do
+      @file_format.should parse_line(@sample_2).as(:access).and_capture(
+        :remote_host    => '1.82.235.29',
+        :remote_logname => nil,
+        :user           => nil,
+        :timestamp      => 20090908075405,
+        :http_status    => 200,
+        :http_method    => 'GET',
+        :http_version   => '1.1',
+        :bytes_sent     => 23414)
+    end
+
+    it "should parse 10 request from fixture access log without warnings" do
+      request_counter.should_receive(:hit!).exactly(10).times
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_file(log_fixture(:apache_common)) do |request| 
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
+      end
+    end
+  end
+
+  context '"Combined" access log parsing' do
+
+    before(:all) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:apache, :combined)
+      @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+      @sample_1 = '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"'
+      @sample_2 = '10.0.1.1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"'
+      @nonsense_sample = 'addasdsasadadssadasd'
+    end
+
+    it "should have a valid language definitions" do
+      @file_format.should be_valid
+    end
+
+    it "should not parse a valid access log line" do
+      @file_format.should_not parse_line(@nonsense_sample)
+    end
+
+    it "should read the correct values from a valid 404 access log line" do
+      @file_format.should parse_line(@sample_1).as(:access).and_capture(
+        :remote_host    => '69.41.0.45',
+        :remote_logname => nil,
+        :user           => nil,
+        :timestamp      => 20090902120240,
+        :http_status    => 404,
+        :http_method    => 'GET',
+        :http_version   => '1.1',
+        :bytes_sent     => 209,
+        :referer        => nil,
+        :user_agent     => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
+    end
+
+    it "should read the correct values from a valid 200 access log line" do
+      @file_format.should parse_line(@sample_2).as(:access).and_capture(
+        :remote_host    => '10.0.1.1',
+        :remote_logname => nil,
+        :user           => nil,
+        :timestamp      => 20090902050833,
+        :http_status    => 200,
+        :http_method    => 'GET',
+        :http_version   => '1.1',
+        :bytes_sent     => 30,
+        :referer        => nil,
+        :user_agent     => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
+    end
+
+    it "should parse 5 request from fixture access log without warnings" do
+      request_counter.should_receive(:hit!).exactly(5).times
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_file(log_fixture(:apache_combined)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
+      end
+    end
+  end
+  
+  context '"Rack" access log parser' do
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:rack)
+      @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+      @sample_1 = '127.0.0.1 - - [16/Sep/2009 07:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453'
+      @sample_2 = '127.0.0.1 - - [01/Oct/2009 07:58:10] "GET / HTTP/1.1" 200 1 0.0045'
+      @nonsense_sample = 'addasdsasadadssadasd'
+    end
+
+    it "should create a kind of an Apache file format" do
+      @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Apache)
+    end
+
+    it "should have a valid language definitions" do
+      @file_format.should be_valid
+    end
+
+    it "should parse a valid access log line with status 500" do
+      @file_format.should parse_line(@sample_1).as(:access).and_capture(
+        :remote_host => '127.0.0.1', :timestamp => 20090916074008, :user => nil,
+        :http_status => 500,         :http_method => 'GET',        :http_version => '1.1',
+        :duration => 0.0453,         :bytes_sent => 63183,         :remote_logname => nil)
+    end
+
+    it "should parse a valid access log line with status 200" do
+      @file_format.should parse_line(@sample_2).as(:access).and_capture(
+        :remote_host => '127.0.0.1', :timestamp => 20091001075810, :user => nil,
+        :http_status => 200,         :http_method => 'GET',        :http_version => '1.1',
+        :duration => 0.0045,         :bytes_sent => 1,             :remote_logname => nil)
+    end
+
+    it "should not parse an invalid access log line" do
+      @file_format.should_not parse_line(@nonsense_sample)
+    end
+
+    it "should parse 2 Apache requests from a sample without warnings" do
+      request_counter.should_receive(:hit!).twice
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_io(log_stream(@sample_1, @nonsense_sample, @sample_2)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
+      end
+    end
+  end
+end

data/spec/unit/file_format/file_format_api_spec.rb

@@ -0,0 +1,69 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat do
+
+  describe ".format_definition" do
+
+    before(:each) do
+      @first_file_format  = Class.new(RequestLogAnalyzer::FileFormat::Base)
+      @second_file_format = Class.new(RequestLogAnalyzer::FileFormat::Base)
+    end
+
+    it "should specify line definitions directly within the file_format" do
+      @first_file_format.format_definition.direct_test :regexp => /test/
+      @first_file_format.should have_line_definition(:direct_test)
+    end
+
+    it "specify lines with a block for the format definition" do
+      @first_file_format.format_definition do |format|
+        format.block_test :regexp => /test (\w+)/, :captures => [{:name => :tester, :type => :string}]
+      end
+
+      @first_file_format.should have_line_definition(:block_test).capturing(:tester)
+    end
+
+    it "should specify a line with a block" do
+      @first_file_format.format_definition.hash_test do |line|
+        line.regexp   = /test/
+        line.captures = []
+      end
+
+      @first_file_format.should have_line_definition(:hash_test)
+    end
+
+    it "should define lines only for its own language" do
+      @first_file_format.format_definition.first   :regexp => /test 123/
+      @second_file_format.format_definition.second :regexp => /test 456/
+
+      @first_file_format.should      have_line_definition(:first)
+      @first_file_format.should_not  have_line_definition(:second)
+      @second_file_format.should_not have_line_definition(:first)
+      @second_file_format.should     have_line_definition(:second)
+    end
+  end
+
+  describe ".load" do
+
+    it "should return an instance of a FileFormat class" do
+      @file_format = RequestLogAnalyzer::FileFormat.load(TestingFormat)
+      @file_format.should be_kind_of(TestingFormat)
+    end
+
+    it "should return itself if it already is a FileFormat::Base instance" do
+      @file_format = RequestLogAnalyzer::FileFormat.load(testing_format)
+      @file_format.should be_kind_of(TestingFormat)
+    end
+
+    it "should load a predefined file format from the /file_format dir" do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+      @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
+    end
+
+    it "should load a provided format file" do
+      format_filename = File.dirname(__FILE__) + '/../../lib/testing_format.rb'
+      @file_format = RequestLogAnalyzer::FileFormat.load(format_filename)
+      @file_format.should be_kind_of(TestingFormat)
+    end
+    
+  end
+end