request-log-analyzer 1.10.1 → 1.11.0

data/spec/unit/file_format/apache_format_spec.rb

@@ -3,29 +3,27 @@ require 'spec_helper'
 describe RequestLogAnalyzer::FileFormat::Apache do
 
   describe '.access_line_definition' do
-    before(:each) do
-      @format_string   = '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T'
-      @line_definition = RequestLogAnalyzer::FileFormat::Apache.access_line_definition(@format_string)
-    end
+    let(:format_string) { '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T' }
+    let(:line_definition) { RequestLogAnalyzer::FileFormat::Apache.access_line_definition(format_string) }
 
     it "should create a Regexp to match the line" do
-      @line_definition.regexp.should be_kind_of(Regexp)
+      line_definition.regexp.should be_kind_of(Regexp)
     end
 
     it "should create a list of captures for the values in the lines" do
-      @line_definition.captures.should have(12).items
+      line_definition.captures.should have(12).items
     end
 
     it "should make it a header line" do
-      @line_definition.should be_header
+      line_definition.should be_header
     end
 
     it "should make it a footer line" do
-      @line_definition.should be_footer
+      line_definition.should be_footer
     end
 
     it "should capture :duration" do
-      @line_definition.captures?(:duration).should be_true
+      line_definition.captures?(:duration).should be_true
     end
   end
   
@@ -47,173 +45,87 @@ describe RequestLogAnalyzer::FileFormat::Apache do
   end
 
   describe '.create' do
+    let(:format_string) { '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"' }
+    subject { RequestLogAnalyzer::FileFormat::Apache.create(format_string) }
 
-    before(:each) do
-      @format_string = '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
-      @format = RequestLogAnalyzer::FileFormat::Apache.create(@format_string)
-    end
-
-    it "should create the :access line definition" do
-      @format.should have_line_definition(:access).capturing(:timestamp, :remote_host, :bytes_sent, :http_method, :path, :http_version, :http_status)
-    end
-
-    it "should be a valid file format" do
-      @format.should be_valid
-    end
-
-    it "should setup report trackers" do
-      @format.report_trackers.should_not be_empty
-    end
+    it { should be_well_formed }
+    it { should have_line_definition(:access).capturing(:timestamp, :remote_host, :bytes_sent, :http_method, :path, :http_version, :http_status) }
+    it { should have(8).report_trackers }
   end
 
   context '"Common" access log parsing' do
-    before(:all) do
-      @file_format     = RequestLogAnalyzer::FileFormat.load(:apache, :common)
-      @log_parser      = RequestLogAnalyzer::Source::LogParser.new(@file_format)
-      @sample_1        = '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223'
-      @sample_2        = '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414'
-      @nonsense_sample = 'addasdsasadadssadasd'
-    end
-
-    it "should have a valid language definitions" do
-      @file_format.should be_valid
-    end
-
-    it "should not parse a valid access log line" do
-      @file_format.should_not parse_line(@nonsense_sample)
-    end
-
-    it "should read the correct values from a valid HTTP/1.0 access log line" do
-      @file_format.should parse_line(@sample_1).as(:access).and_capture(
-        :remote_host    => '1.129.119.13',
-        :remote_logname => nil,
-        :user           => nil,
-        :timestamp      => 20090908075409,
-        :http_status    => 200,
-        :http_method    => 'GET',
-        :http_version   => '1.0',
-        :bytes_sent     => 8223)
-    end
-
-    it "should read the correct values from a valid 200 access log line" do
-      @file_format.should parse_line(@sample_2).as(:access).and_capture(
-        :remote_host    => '1.82.235.29',
-        :remote_logname => nil,
-        :user           => nil,
-        :timestamp      => 20090908075405,
-        :http_status    => 200,
-        :http_method    => 'GET',
-        :http_version   => '1.1',
-        :bytes_sent     => 23414)
+    subject { RequestLogAnalyzer::FileFormat.load(:apache, :common) } 
+    
+    it { should be_well_formed }
+    it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent) }
+    it { should have(6).report_trackers }
+    
+    describe '#parse_line' do
+      
+      let(:sample1) { '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223' }
+      let(:sample2) { '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414' }
+      
+      it { should parse_line(sample1, 'a sample line').and_capture(
+            :remote_host    => '1.129.119.13', :remote_logname => nil, :user        => nil,
+            :timestamp      => 20090908075409, :http_status    => 200, :http_method => 'GET',
+            :http_version   => '1.0',          :bytes_sent     => 8223)
+      }
+      
+      it { should parse_line(sample2, 'another sample line').and_capture(
+            :remote_host    => '1.82.235.29',  :remote_logname => nil, :user        => nil,
+            :timestamp      => 20090908075405, :http_status    => 200, :http_method => 'GET',
+            :http_version   => '1.1',          :bytes_sent     => 23414)
+      }
+      
+      it { should_not parse_line('nonsense', 'a nonsense line')}
     end
-
-    it "should parse 10 request from fixture access log without warnings" do
-      request_counter.should_receive(:hit!).exactly(10).times
-      @log_parser.should_not_receive(:warn)
+    
+    describe '#parse_io' do
+      let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
       
-      @log_parser.parse_file(log_fixture(:apache_common)) do |request| 
-        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
+      it "should parse a log snippet successfully without warnings" do
+        log_parser.should_receive(:handle_request).exactly(10).times
+        log_parser.should_not_receive(:warn)
+        log_parser.parse_file(log_fixture(:apache_common))
       end
     end
   end
 
   context '"Combined" access log parsing' do
-
-    before(:all) do
-      @file_format = RequestLogAnalyzer::FileFormat.load(:apache, :combined)
-      @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
-      @sample_1 = '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"'
-      @sample_2 = '10.0.1.1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"'
-      @nonsense_sample = 'addasdsasadadssadasd'
-    end
-
-    it "should have a valid language definitions" do
-      @file_format.should be_valid
-    end
-
-    it "should not parse a valid access log line" do
-      @file_format.should_not parse_line(@nonsense_sample)
-    end
-
-    it "should read the correct values from a valid 404 access log line" do
-      @file_format.should parse_line(@sample_1).as(:access).and_capture(
-        :remote_host    => '69.41.0.45',
-        :remote_logname => nil,
-        :user           => nil,
-        :timestamp      => 20090902120240,
-        :http_status    => 404,
-        :http_method    => 'GET',
-        :http_version   => '1.1',
-        :bytes_sent     => 209,
-        :referer        => nil,
-        :user_agent     => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
-    end
-
-    it "should read the correct values from a valid 200 access log line" do
-      @file_format.should parse_line(@sample_2).as(:access).and_capture(
-        :remote_host    => '10.0.1.1',
-        :remote_logname => nil,
-        :user           => nil,
-        :timestamp      => 20090902050833,
-        :http_status    => 200,
-        :http_method    => 'GET',
-        :http_version   => '1.1',
-        :bytes_sent     => 30,
-        :referer        => nil,
-        :user_agent     => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
-    end
-
-    it "should parse 5 request from fixture access log without warnings" do
-      request_counter.should_receive(:hit!).exactly(5).times
-      @log_parser.should_not_receive(:warn)
+    subject { RequestLogAnalyzer::FileFormat.load(:apache, :combined) } 
+    
+    it { should be_well_formed }
+    it { should have_line_definition(:access).capturing(:remote_host, :remote_logname, :user, :timestamp, :http_status, :http_method, :http_version, :bytes_sent, :referer, :user_agent) }
+    it { should have(8).report_trackers }
+    
+    describe '#parse_line' do
+      let(:sample1) { '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"' }
+      let(:sample2) { '0:0:0:0:0:0:0:1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"' }
       
-      @log_parser.parse_file(log_fixture(:apache_combined)) do |request|
-        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
-      end
-    end
-  end
-  
-  context '"Rack" access log parser' do
-    before(:each) do
-      @file_format = RequestLogAnalyzer::FileFormat.load(:rack)
-      @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
-      @sample_1 = '127.0.0.1 - - [16/Sep/2009 07:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453'
-      @sample_2 = '127.0.0.1 - - [01/Oct/2009 07:58:10] "GET / HTTP/1.1" 200 1 0.0045'
-      @nonsense_sample = 'addasdsasadadssadasd'
-    end
-
-    it "should create a kind of an Apache file format" do
-      @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Apache)
-    end
-
-    it "should have a valid language definitions" do
-      @file_format.should be_valid
-    end
-
-    it "should parse a valid access log line with status 500" do
-      @file_format.should parse_line(@sample_1).as(:access).and_capture(
-        :remote_host => '127.0.0.1', :timestamp => 20090916074008, :user => nil,
-        :http_status => 500,         :http_method => 'GET',        :http_version => '1.1',
-        :duration => 0.0453,         :bytes_sent => 63183,         :remote_logname => nil)
-    end
-
-    it "should parse a valid access log line with status 200" do
-      @file_format.should parse_line(@sample_2).as(:access).and_capture(
-        :remote_host => '127.0.0.1', :timestamp => 20091001075810, :user => nil,
-        :http_status => 200,         :http_method => 'GET',        :http_version => '1.1',
-        :duration => 0.0045,         :bytes_sent => 1,             :remote_logname => nil)
-    end
-
-    it "should not parse an invalid access log line" do
-      @file_format.should_not parse_line(@nonsense_sample)
+      it { should parse_line(sample1, 'with IPv4 address').and_capture(
+            :remote_host  => '69.41.0.45',   :remote_logname => nil, :user        => nil,
+            :timestamp    => 20090902120240, :http_status    => 404, :http_method => 'GET',
+            :http_version => '1.1',          :bytes_sent     => 209, :referer     => nil,
+            :user_agent   => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)')
+      }
+      
+      it { should parse_line(sample2, 'with IPv6 address').and_capture(
+            :remote_host  => '0:0:0:0:0:0:0:1', :remote_logname => nil, :user        => nil,
+            :timestamp    => 20090902050833,    :http_status    => 200, :http_method => 'GET',
+            :http_version => '1.1',             :bytes_sent     => 30,  :referer     => nil,
+            :user_agent   => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9')
+      }
+      
+      it { should_not parse_line('nonsense', 'a nonsense line')}
     end
-
-    it "should parse 2 Apache requests from a sample without warnings" do
-      request_counter.should_receive(:hit!).twice
-      @log_parser.should_not_receive(:warn)
+    
+    describe '#parse_io' do
+      let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) }
       
-      @log_parser.parse_io(log_stream(@sample_1, @nonsense_sample, @sample_2)) do |request|
-        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Apache::Request)
+      it "should parse a log snippet successfully without warnings" do
+        log_parser.should_receive(:handle_request).exactly(5).times
+        log_parser.should_not_receive(:warn)
+        log_parser.parse_file(log_fixture(:apache_combined))
       end
     end
   end

data/spec/unit/file_format/common_regular_expressions_spec.rb

@@ -3,51 +3,76 @@ require 'spec_helper'
 describe RequestLogAnalyzer::FileFormat::CommonRegularExpressions do
   
   include RequestLogAnalyzer::FileFormat::CommonRegularExpressions
-  
+
   describe '.timestamp' do
-    
     it "should parse timestamps with a given format" do
-      timestamp('%Y-%m-%dT%H:%M:%S%z').should =~ '2009-12-03T00:12:37+0100'
-      timestamp('%Y-%m-%dT%H:%M:%S%z').should_not =~ '2009-12-03 00:12:37+0100'
-      timestamp('%Y-%m-%dT%H:%M:%S%z').should_not =~ '2009-12-03T00:12:37'
+      anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should =~ '2009-12-03T00:12:37+0100'
+      anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03 00:12:37+0100'
+      anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03T00:12:37'
     end
   end
+
   
-  describe '.ip_address' do
+  describe '.hostname' do
+    it "should parse hostnames successfully" do
+      anchored(hostname).should =~ 'railsdoctors.com'
+      anchored(hostname).should =~ 'www.rails-doctors.com'
+      anchored(hostname).should =~ 'hostname.co.uk'
+      anchored(hostname).should =~ 'localhost'
     
+      anchored(hostname).should_not =~ '192.168.0.1'
+      anchored(hostname).should_not =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
+      anchored(hostname).should_not =~ 'railsdoctors.'
+    end
+  end
+  
+  describe '.ip_address' do
     it "should parse IPv4 addresses" do
-      ip_address.should =~ '127.0.0.1'
-      ip_address.should =~ '255.255.255.255'
+      anchored(ip_address).should =~ '127.0.0.1'
+      anchored(ip_address).should =~ '255.255.255.255'
       
-      ip_address.should_not =~ '2552.2552.2552.2552'
-      ip_address.should_not =~ '127001'
-      ip_address.should_not =~ ''
-      ip_address.should_not =~ '-'
-      ip_address.should_not =~ 'sub-host.domain.tld'
+      anchored(ip_address).should_not =~ '2552.2552.2552.2552'
+      anchored(ip_address).should_not =~ '127001'
+      anchored(ip_address).should_not =~ ''
+      anchored(ip_address).should_not =~ '-'
+      anchored(ip_address).should_not =~ 'sub-host.domain.tld'
     end
     
     it "should pase IPv6 addresses" do
-      ip_address.should =~ '::1'
-      ip_address.should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
-      ip_address.should =~ '3ffe:1900:4545:3:200:f8ff:127.0.0.1'
-      ip_address.should =~ '::3:200:f8ff:127.0.0.1'
+      anchored(ip_address).should =~ '::1'
+      anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
+      anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:127.0.0.1'
+      anchored(ip_address).should =~ '::3:200:f8ff:127.0.0.1'
+      anchored(ip_address).should =~ '0:0:0:0:0:0:0:1'
       
-      ip_address.should_not =~ 'qqqq:wwww:eeee:3q:200:wf8ff:fe21:67cf'
-      ip_address.should_not =~ '3ffe44:1900f:454545:3:200:f8ff:ffff:5432'
+      anchored(ip_address).should_not =~ 'qqqq:wwww:eeee:3q:200:wf8ff:fe21:67cf'
+      anchored(ip_address).should_not =~ '3ffe44:1900f:454545:3:200:f8ff:ffff:5432'
     end
     
     it "should allow blank if true is given as parameter" do
-      /^#{ip_address(true)}$/.should =~ ''
-      /^#{ip_address(true)}$/.should_not =~ ' '
+      anchored(ip_address(true)).should =~ ''
+      anchored(ip_address(true)).should_not =~ ' '
     end
     
     it "should allow a nil substitute if a string is given as parameter" do
-      /^#{ip_address('-')}$/.should =~ '-'
-      /^#{ip_address('-')}$/.should_not =~ ' -'
-      /^#{ip_address('-')}$/.should_not =~ '--'
-      /^#{ip_address('-')}$/.should_not =~ ''
+      anchored(ip_address('-')).should =~ '-'
+      anchored(ip_address('-')).should_not =~ ' -'
+      anchored(ip_address('-')).should_not =~ '--'
+      anchored(ip_address('-')).should_not =~ ''
     end
-    
   end
+  
+  
+  describe '.hostname_or_ip_address' do
+    it "should parse either hostnames or ip addresses" do
+      anchored(hostname_or_ip_address).should =~ 'railsdoctors.com'
+      anchored(hostname_or_ip_address).should =~ 'hostname.co.uk'
+      anchored(hostname_or_ip_address).should =~ 'localhost'
+      anchored(hostname_or_ip_address).should =~ '192.168.0.1'
+      anchored(hostname_or_ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf'
+      
+      anchored(hostname_or_ip_address).should_not =~ 'railsdoctors.'
+    end
+  end  
 end
 

data/spec/unit/file_format/delayed_job21_format_spec.rb

@@ -2,42 +2,36 @@ require 'spec_helper'
 
 describe RequestLogAnalyzer::FileFormat::DelayedJob do
 
-  it "should be a valid file format" do
-    RequestLogAnalyzer::FileFormat.load(:delayed_job).should be_valid
-  end
+  subject { RequestLogAnalyzer::FileFormat.load(:delayed_job21) }
+  
+  it { should be_well_formed }
+  it { should have_line_definition(:job_lock).capturing(:timestamp, :job, :host, :pid) }
+  it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid) }
+  it { should have(4).report_trackers }
 
   describe '#parse_line' do
     
-    before(:each) do
-      @file_format = RequestLogAnalyzer::FileFormat.load(:delayed_job21)
-    end
+    let(:job_lock_sample1) { "2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob" } 
+    let(:job_lock_sample2) { "2010-05-17T17:37:34+0000: [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob" } 
+    let(:job_completed_sample1) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676' }
 
-    it "should parse a :job_lock line correctly" do
-      line = "2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob"
-      @file_format.should parse_line(line).as(:job_lock).and_capture(:timestamp => 20100517173734,
-                                    :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888)
-    end
+    it { should parse_line(job_lock_sample1, 'with a single worker').as(:job_lock).and_capture(
+          :timestamp => 20100517173734, :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888) }
 
-    it "should parse a :job_lock line correctly when the worker is one of many" do
-      line = "2010-05-17T17:37:34+0000: [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob"
-      @file_format.should parse_line(line).as(:job_lock).and_capture(:timestamp => 20100517173734,
-                                    :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888)
-    end
+    it { should parse_line(job_lock_sample2, 'with multiple workers').as(:job_lock).and_capture(
+          :timestamp => 20100517173734, :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888) }
 
-    it "should parse a :job_completed line correctly" do
-      line = '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676'
-      @file_format.should parse_line(line).as(:job_completed).and_capture(:timestamp => 20100517173735,
-                                    :duration => 1.0676, :host => 'hostname.co.uk', :pid => 11888, :job => 'S3FileJob')
-    end
+    it { should parse_line(job_completed_sample1).as(:job_completed).and_capture(
+          :timestamp => 20100517173735, :duration => 1.0676, :host => 'hostname.co.uk', :pid => 11888, :job => 'S3FileJob') }
+          
+    it { should_not parse_line('nonsense', 'a nonsense line') }
   end
   
   describe '#parse_io' do
-    before(:each) do
-      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:delayed_job21))
-    end
+    let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 
     
     it "should parse a batch of completed jobs without warnings" do
-      fragment = <<-EOLOG
+      fragment = log_snippet(<<-EOLOG)
         2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
         2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
         2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676
@@ -49,12 +43,9 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
         2010-05-19T11:47:26+0000: Exiting...
       EOLOG
 
-      request_counter.should_receive(:hit!).exactly(3).times
-      @log_parser.should_not_receive(:warn)
-
-      @log_parser.parse_io(StringIO.new(fragment)) do |request|
-        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::DelayedJob21::Request)
-      end
+      log_parser.should_receive(:handle_request).exactly(3).times
+      log_parser.should_not_receive(:warn)
+      log_parser.parse_io(fragment)
     end
   end
 end