request-log-analyzer 1.2.8 → 1.2.9

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
   
   # The current version of request-log-analyzer.
   # This will be diplayed in output reports etc.  
-  VERSION = "1.2.8"
+  VERSION = "1.2.9"
   
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/lib/request_log_analyzer/file_format/apache.rb

@@ -16,11 +16,20 @@ module RequestLogAnalyzer::FileFormat
 
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
-      'h' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :ip_address, :type => :string}] },
+      'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
+      'a' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :remote_ip, :type => :string}] },
+      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :integer}] },
+      'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
+      'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
+      'T' => { :regexp => '((?:\d+(?:\.\d+)?)|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
       't' => { :regexp => '\[([^\]]{26})\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
       's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
+      'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
       'r' => { :regexp => '([A-Z]+) ([^\s]+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
-                       {:name => :path, :type => :string}, {:name => :http_version, :type => :string}]}
+                       {:name => :path, :type => :string}, {:name => :http_version, :type => :string}]},
+      'i' => { 'Referer'    => { :regexp => '([^\s]+)', :captures => [{:name => :referer, :type => :nillable_string}] },
+               'User-agent' => { :regexp => '(.*)',     :captures => [{:name => :user_agent, :type => :user_agent}] }
+             }
     }
 
     # Creates the Apache log format language based on a Apache log format string.
@@ -41,9 +50,13 @@ module RequestLogAnalyzer::FileFormat
       format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z]))?/) do |literal, arg, variable|
 
         line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
+        
         if variable
           # Check if we recognize the log directive
-          if directive = LOG_DIRECTIVES[variable]
+          directive = LOG_DIRECTIVES[variable]
+          directive = directive[arg] if directive && arg
+
+          if directive
             line_regexp << directive[:regexp]   # Parse the value of the directive
             captures    += directive[:captures] # Add the directive's information to the captures
           else
@@ -68,6 +81,13 @@ module RequestLogAnalyzer::FileFormat
       analyze.frequency :category => :http_status, :amount => 20, :title => "HTTP statuses" if line_definition.captures?(:http_status)
       analyze.frequency :category => :path, :amount => 20, :title => "Most popular URIs"    if line_definition.captures?(:path)
 
+      analyze.frequency :category => :user_agent, :amount => 20, :title => "User agents"    if line_definition.captures?(:user_agent)
+      analyze.frequency :category => :referer,    :amount => 20, :title => "Referers"       if line_definition.captures?(:referer)
+
+      if line_definition.captures?(:path) && line_definition.captures?(:duration)
+        analyze.duration :duration => :duration, :category => :path , :title => 'Request duration'
+      end
+
       return analyze.trackers
     end
 
@@ -82,6 +102,14 @@ module RequestLogAnalyzer::FileFormat
         d = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})/.match(value).captures
         "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
       end
+      
+      def convert_user_agent(value, definition)
+        value # TODO
+      end
+      
+      def convert_nillable_string(value, definition)
+        value == '-' ? nil : value
+      end
     end
   end
 end

data/request-log-analyzer.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'request-log-analyzer'
-  s.version = "1.2.8"
+  s.version = "1.2.9"
   s.date    = "2009-09-08"
   
   s.rubyforge_project = 'r-l-a'
@@ -31,6 +31,6 @@ Gem::Specification.new do |s|
   s.email    = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
   s.homepage = 'http://railsdoctors.com'
 
-  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/file_format/rails_development.rb spec/lib/macros.rb spec/fixtures/merb_prefixed.log tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/fixtures/decompression.log.bz2 lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb spec/fixtures/rails_unordered.log bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/matchers.rb spec/fixtures/test_order.log lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb spec/lib/testing_format.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb Rakefile spec/unit/filter/filter_spec.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/test_language_combined.log spec/fixtures/decompression.tar.gz spec/unit/filter/field_filter_spec.rb spec/spec.opts lib/request_log_analyzer/aggregator/database.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/apache.log spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/lib/helpers.rb spec/fixtures/rails_1x.log spec/lib/mocks.rb spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/aggregator/database_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/source/database.rb tasks/github-gem.rake lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log spec/fixtures/syslog_1x.log LICENSE spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
+  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/file_format/rails_development.rb spec/lib/macros.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/fixtures/decompression.log.bz2 lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb spec/fixtures/rails_unordered.log bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/matchers.rb spec/fixtures/test_order.log lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb spec/lib/testing_format.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb Rakefile spec/unit/filter/filter_spec.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/test_language_combined.log spec/fixtures/decompression.tar.gz spec/unit/filter/field_filter_spec.rb spec/spec.opts lib/request_log_analyzer/aggregator/database.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/lib/helpers.rb spec/fixtures/rails_1x.log spec/lib/mocks.rb spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/aggregator/database_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/source/database.rb tasks/github-gem.rake lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log spec/fixtures/syslog_1x.log LICENSE spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
   s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/aggregator/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
 end

data/spec/fixtures/apache_common.log

@@ -0,0 +1,10 @@
+1.56.136.157 - - [08/Sep/2009:07:54:07 -0400] "GET /form_configurations/show.flashxml HTTP/1.1" 304 -
+1.72.56.31 - - [08/Sep/2009:07:54:07 -0400] "GET /users/M17286/projects.xml?per_page=8000 HTTP/1.1" 404 1
+1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414
+1.56.136.157 - - [08/Sep/2009:07:54:08 -0400] "GET /designs/20110457.flashxml HTTP/1.1" 304 -
+1.56.136.157 - - [08/Sep/2009:07:54:08 -0400] "GET /object_libraries/1.flashxml HTTP/1.1" 304 -
+1.249.68.72 - - [08/Sep/2009:07:54:09 -0400] "GET /projects/18182835-my-first-project HTTP/1.1" 200 10435
+1.108.106.20 - - [08/Sep/2009:07:54:09 -0400] "GET /login HTTP/1.1" 200 9522
+1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223
+1.158.151.25 - - [08/Sep/2009:07:54:09 -0400] "GET /projects/18236114-home/floors/18252888-/designs/19406560-home-1 HTTP/1.1" 200 10434
+1.29.137.1 - - [08/Sep/2009:07:54:10 -0400] "GET /assets/custom/blah/images/print_logo.jpg HTTP/1.1" 200 10604

data/spec/integration/command_line_usage_spec.rb

@@ -55,13 +55,13 @@ describe RequestLogAnalyzer, 'running from command line' do
     output.all? { |line| /^[\x00-\x7F]*$/ =~ line }.should be_true
   end
 
-  it "should parse a Merb file if --apache-format is set" do
+  it "should parse a Merb file if --format merb is set" do
     output = run("#{log_fixture(:merb)} --format merb")
     output.detect { |line| /Parsed requests\:\s*11/ =~ line }.should_not be_nil
   end
 
-  it "should parse a Apache access log file if --format merb is set" do
-    output = run("#{log_fixture(:apache)} --apache-format combined")
+  it "should parse a Apache access log file if --apache-format is set" do
+    output = run("#{log_fixture(:apache_combined)} --apache-format combined")
     output.detect { |line| /Parsed requests\:\s*5/ =~ line }.should_not be_nil
   end
 

data/spec/unit/file_format/apache_format_spec.rb

@@ -4,7 +4,7 @@ describe RequestLogAnalyzer::FileFormat::Apache do
 
   describe '.access_line_definition' do
     before(:each) do
-      @format_string   = '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
+      @format_string   = '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i" %T'
       @line_definition = RequestLogAnalyzer::FileFormat::Apache.access_line_definition(@format_string)
     end
 
@@ -13,7 +13,7 @@ describe RequestLogAnalyzer::FileFormat::Apache do
     end
 
     it "should create a list of captures for the values in the lines" do
-      @line_definition.captures.should be_kind_of(Array)
+      @line_definition.captures.should have(12).items
     end
 
     it "should make it a header line" do
@@ -23,6 +23,10 @@ describe RequestLogAnalyzer::FileFormat::Apache do
     it "should make it a footer line" do
       @line_definition.should be_footer
     end
+    
+    it "should capture :duration" do
+      @line_definition.captures?(:duration).should be_true
+    end
   end
 
   describe '.create' do
@@ -33,7 +37,7 @@ describe RequestLogAnalyzer::FileFormat::Apache do
     end
 
     it "should create the :access line definition" do
-      @format.should have_line_definition(:access).capturing(:timestamp, :ip_address, :http_method, :path, :http_version, :http_status)
+      @format.should have_line_definition(:access).capturing(:timestamp, :remote_host, :bytes_sent, :http_method, :path, :http_version, :http_status)
     end
 
     it "should be a valid file format" do
@@ -45,12 +49,64 @@ describe RequestLogAnalyzer::FileFormat::Apache do
     end
   end
 
-  context 'log parsing' do
+  context '"Common" access log parsing' do
+    before(:all) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:apache, :common)
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+      @sample_1 = '1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223'
+      @sample_2 = '1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414'
+    end
 
-    before(:each) do
-      @file_format = RequestLogAnalyzer::FileFormat.load(:apache)
+    it "should have a valid language definitions" do
+      @file_format.should be_valid
+    end
+    
+    it "should parse a valid access log line" do
+      @file_format.line_definitions[:access].matches(@sample_1).should be_kind_of(Hash)
+    end
+
+    it "should not parse a valid access log line" do
+      @file_format.line_definitions[:access].matches('addasdsasadadssadasd').should be_false
+    end
+    
+    it "should read the correct values from a valid HTTP/1.0 access log line" do
+      @log_parser.parse_io(@sample_1) do |request|
+        request[:remote_host].should  == '1.129.119.13'
+        request[:timestamp].should    == 20090908075409
+        request[:http_status].should  == 200
+        request[:http_method].should  == 'GET'
+        request[:http_version].should == '1.0'
+        request[:bytes_sent].should   == 8223
+        request[:user].should         == nil
+      end
+    end
+    
+    it "should read the correct values from a valid 200 access log line" do
+      @log_parser.parse_io(@sample_2) do |request|
+        request[:remote_host].should  == '1.82.235.29'
+        request[:timestamp].should    == 20090908075405
+        request[:http_status].should  == 200
+        request[:http_method].should  == 'GET'
+        request[:http_version].should == '1.1'
+        request[:bytes_sent].should   == 23414
+        request[:user].should         == nil
+      end
+    end    
+
+    it "should parse 10 request from fixture access log" do
+      counter = mock('counter')
+      counter.should_receive(:hit!).exactly(10).times
+      @log_parser.parse_file(log_fixture(:apache_common)) { counter.hit! }
+    end
+  end
+ 
+  context '"Combined" access log parsing' do
+
+    before(:all) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:apache, :combined)
       @log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
-      @sample = '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"'
+      @sample_1 = '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"'
+      @sample_2 = '10.0.1.1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"'
     end
 
     it "should have a valid language definitions" do
@@ -58,23 +114,45 @@ describe RequestLogAnalyzer::FileFormat::Apache do
     end
 
     it "should parse a valid access log line" do
-      @file_format.line_definitions[:access].matches(@sample).should be_kind_of(Hash)
+      @file_format.line_definitions[:access].matches(@sample_1).should be_kind_of(Hash)
+    end
+
+    it "should not parse a valid access log line" do
+      @file_format.line_definitions[:access].matches('addasdsasadadssadasd').should be_false
     end
 
-    it "should read the correct values from a valid access log line" do
-      @log_parser.parse_io(@sample) do |request|
-        request[:ip_address].should   == '69.41.0.45'
+    it "should read the correct values from a valid 404 access log line" do
+      @log_parser.parse_io(@sample_1) do |request|
+        request[:remote_host].should  == '69.41.0.45'
         request[:timestamp].should    == 20090902120240
         request[:http_status].should  == 404
         request[:http_method].should  == 'GET'
         request[:http_version].should == '1.1'
+        request[:bytes_sent].should   == 209
+        request[:referer].should      == nil
+        request[:user].should         == nil
+        request[:user_agent].should   == 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)'
       end
     end
+    
+    it "should read the correct values from a valid 200 access log line" do
+      @log_parser.parse_io(@sample_2) do |request|
+        request[:remote_host].should  == '10.0.1.1'
+        request[:timestamp].should    == 20090902050833
+        request[:http_status].should  == 200
+        request[:http_method].should  == 'GET'
+        request[:http_version].should == '1.1'
+        request[:bytes_sent].should   == 30
+        request[:referer].should      == nil
+        request[:user].should         == nil
+        request[:user_agent].should   == 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9'
+      end
+    end    
   
     it "should parse 5 request from fixture access log" do
       counter = mock('counter')
       counter.should_receive(:hit!).exactly(5).times
-      @log_parser.parse_file(log_fixture(:apache)) { counter.hit! }
+      @log_parser.parse_file(log_fixture(:apache_combined)) { counter.hit! }
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.2.8
+  version: 1.2.9
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -51,6 +51,8 @@ files:
 - spec/fixtures/rails_22_cached.log
 - lib/request_log_analyzer/file_format/rails_development.rb
 - spec/lib/macros.rb
+- spec/fixtures/apache_combined.log
+- spec/fixtures/apache_common.log
 - spec/fixtures/merb_prefixed.log
 - tasks/request_log_analyzer.rake
 - spec/unit/file_format/file_format_api_spec.rb
@@ -101,7 +103,6 @@ files:
 - spec/fixtures/decompression.tgz
 - spec/unit/tracker/timespan_tracker_spec.rb
 - spec/unit/tracker/hourly_spread_spec.rb
-- spec/fixtures/apache.log
 - spec/fixtures/header_and_footer.log
 - lib/cli/tools.rb
 - lib/request_log_analyzer/file_format/merb.rb