request-log-analyzer 1.5.1 → 1.5.2

data/lib/cli/tools.rb

@@ -2,24 +2,27 @@
 # If it is not possible to to so, it returns the default_width.
 # <tt>default_width</tt> Defaults to 81
 def terminal_width(default_width = 81)
-  tiocgwinsz = 0x5413
-  data = [0, 0, 0, 0].pack("SSSS")
-  if @out.ioctl(tiocgwinsz, data) >= 0
-    rows, cols, xpixels, ypixels = data.unpack("SSSS")
-    raise unless cols > 0
-    cols
-  else
-    raise
-  end
-rescue
+  
   begin
-    IO.popen('stty -a 2>&1') do |pipe|
-      column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
-      raise unless column_line
-      $1.to_i
+    tiocgwinsz = 0x5413
+    data = [0, 0, 0, 0].pack("SSSS")
+    if !RUBY_PLATFORM.include?('java') && @out.ioctl(tiocgwinsz, data) >= 0 # JRuby crashes on ioctl
+      rows, cols, xpixels, ypixels = data.unpack("SSSS")
+      raise unless cols > 0
+      cols
+    else
+      raise
+    end
+  rescue 
+    begin
+      IO.popen('stty -a 2>&1') do |pipe|
+        column_line = pipe.detect { |line| /(\d+) columns/ =~ line }
+        raise unless column_line
+        $1.to_i
+      end
+    rescue
+      default_width
     end
-  rescue
-    default_width
   end
 end
 

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -9,7 +9,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :access do |line|
       line.header = true
       line.footer = true
-      line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
+      line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] ([\d\.:]+) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
       line.captures << { :name => :bucket_owner,    :type => :string } <<
                        { :name => :bucket,          :type => :string } <<
                        { :name => :timestamp,       :type => :timestamp } <<

data/lib/request_log_analyzer/file_format/apache.rb

@@ -27,7 +27,7 @@ module RequestLogAnalyzer::FileFormat
     LOG_DIRECTIVES = {
       '%' => { :regexp => '%', :captures => [] },
       'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
-      'a' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :remote_ip, :type => :string}] },
+      'a' => { :regexp => '([\d\.:]+)', :captures => [{:name => :remote_ip, :type => :string}] },
       'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
       'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
       'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
@@ -57,11 +57,12 @@ module RequestLogAnalyzer::FileFormat
     def self.access_line_definition(format_string)
       format_string ||= :common
       format_string   = LOG_FORMAT_DEFAULTS[format_string.to_sym] || format_string
-
+      
+      
       line_regexp = ''
       captures    = []
       format_string.scan(/([^%]*)(?:%(?:\{([^\}]+)\})?>?([A-Za-z%]))?/) do |literal, arg, variable|
-
+        
         line_regexp << Regexp.quote(literal) # Make sure to parse the literal before the directive
 
         if variable
@@ -78,7 +79,7 @@ module RequestLogAnalyzer::FileFormat
           end
         end
       end
-
+      
       # Return a new line definition object
       return RequestLogAnalyzer::LineDefinition.new(:access, :regexp => Regexp.new(line_regexp),
                                         :captures => captures, :header => true, :footer => true)

data/lib/request_log_analyzer/file_format/merb.rb

@@ -14,7 +14,7 @@ module RequestLogAnalyzer::FileFormat
     end
 
     # ~ Params: {"action"=>"create", "controller"=>"session"}
-    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
+    # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy"}
     line_definition :params do |line|
       line.teaser = /Params\:\ /
       line.regexp = /Params\:\ (\{.+\})/

data/lib/request_log_analyzer/file_format/rails.rb

@@ -86,7 +86,7 @@ module RequestLogAnalyzer::FileFormat
     LINE_DEFINITIONS = {
       :processing => RequestLogAnalyzer::LineDefinition.new(:processing, :header => true,
             :teaser   => /Processing /,
-            :regexp   => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
+            :regexp   => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for ([\d\.:]+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
             :captures => [{ :name => :controller, :type  => :string },
                           { :name => :action,     :type  => :string },
                           { :name => :format,     :type  => :string, :default => 'html' },

data/lib/request_log_analyzer/file_format.rb

@@ -85,6 +85,10 @@ module RequestLogAnalyzer::FileFormat
     score -= parser.warnings * 3
     score += parser.parsed_lines * 1
     score += parser.parsed_requests * 10
+
+    # As Apache matches several simular formats, subtracting 1 will make a specific matcher have a higher score
+    score -= 1 if parser.file_format.class == RequestLogAnalyzer::FileFormat::Apache
+    
     score
   end
 

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
 
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.5.1"
+  VERSION = "1.5.2"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.5.1"
-  s.date    = "2009-11-19"
+  s.version = "1.5.2"
+  s.date    = "2009-12-02"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -11,13 +11,14 @@ Gem::Specification.new do |s|
   s.executables        = ['request-log-analyzer']
   s.default_executable = 'request-log-analyzer'
 
-  s.summary     = "A command line tool to analyze request logs for Apache, Rails, Merb and other web application servers"
+  s.summary     = "A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers"
   s.description = <<-eos
-    Request log analyzer's purpose is to find ot how your web application is being used and to focus your optimization efforts.
-    This tool will parse all requests in the application's log file and aggregate the information. Once it is finished parsing
-    the log file(s), it will show the requests that take op most server time using various metrics. It can also insert all
-    parsed request information into a database so you can roll your own analysis. It supports Rails- and Merb-based applications
-    and Apache access log files out of the box, but file formats of other applications can easily be supported by supplying an
+    Request log analyzer's purpose is to find out how your web application is being used, how it performs and to
+    focus your optimization efforts. This tool will parse all requests in the application's log file and aggregate the 
+    information. Once it is finished parsing the log file(s), it will show the requests that take op most server time 
+    using various metrics. It can also insert all parsed request information into a database so you can roll your own
+    analysis. It supports Rails-, Merb- and Rack-based applications logs, Apache and Amazon S3 access logs and MySQL 
+    slow query logs out of the box, but file formats of other applications can easily be supported by supplying an 
     easy to write log file format definition.
   eos
 
@@ -35,6 +36,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(lib/request_log_analyzer/output/html.rb spec/unit/tracker/hourly_spread_spec.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/unit/tracker/count_tracker_spec.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/integration/munin_plugins_rails_spec.rb spec/lib/helpers.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb spec/fixtures/test_language_combined.log lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml spec/unit/request_spec.rb lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/tracker/count.rb lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log lib/request_log_analyzer/file_format/rails.rb Rakefile spec/unit/mailer_spec.rb lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
-  s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/filter/filter_spec.rb spec/unit/tracker/count_tracker_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
+  s.files      = %w(spec/unit/tracker/hourly_spread_spec.rb lib/request_log_analyzer/output/html.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/unit/file_format/rack_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/unit/tracker/count_tracker_spec.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/integration/munin_plugins_rails_spec.rb spec/lib/helpers.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb spec/fixtures/test_language_combined.log lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml spec/unit/request_spec.rb lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/tracker/count.rb lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log lib/request_log_analyzer/file_format/rails.rb Rakefile spec/unit/mailer_spec.rb lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/fixtures/sinatra.log spec/unit/filter/field_filter_spec.rb spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
+  s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/filter/filter_spec.rb spec/unit/tracker/count_tracker_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
 end

data/spec/fixtures/sinatra.log

@@ -0,0 +1,99 @@
+== Sinatra/0.9.4 has taken the stage on 4567 for development with backup from Mongrel
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET / HTTP/1.1" 200 5056 0.0241
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /javascripts/cufon-yui.js HTTP/1.1" 304 - 0.0016
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /javascripts/CartoGothic_Std_400-CartoGothic_Std_700-CartoGothic_Std_italic_400-CartoGothic_Std_italic_700.font.js HTTP/1.1" 304 - 0.0013
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /css/stylesheet.css HTTP/1.1" 304 - 0.0014
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/rails_doctors_logo.png HTTP/1.1" 304 - 0.0039
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/footer-bg.png HTTP/1.1" 304 - 0.0028
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/nav-bg.png HTTP/1.1" 304 - 0.0023
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/radial-bg.png HTTP/1.1" 304 - 0.0019
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/box-bg.png HTTP/1.1" 304 - 0.0015
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/performance_train.png HTTP/1.1" 304 - 0.0011
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/lock.png HTTP/1.1" 304 - 0.0025
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/contact.png HTTP/1.1" 304 - 0.0015
+127.0.0.1 - - [23/Nov/2009 22:46:22] "GET /images/rla.png HTTP/1.1" 304 - 0.0008
+127.0.0.1 - - [23/Nov/2009 22:46:24] "GET /index.html HTTP/1.1" 200 5056 0.0216
+127.0.0.1 - - [23/Nov/2009 22:46:25] "GET /blog.html HTTP/1.1" 200 8156 0.0115
+127.0.0.1 - - [23/Nov/2009 22:46:26] "GET /performance.html HTTP/1.1" 200 4751 0.0134
+127.0.0.1 - - [23/Nov/2009 22:46:26] "GET /blog.html HTTP/1.1" 200 8156 0.0504
+127.0.0.1 - - [23/Nov/2009 22:46:27] "GET /security.html HTTP/1.1" 200 4503 0.0119
+127.0.0.1 - - [23/Nov/2009 22:46:27] "GET /performance.html HTTP/1.1" 200 4751 0.0206
+127.0.0.1 - - [23/Nov/2009 22:46:29] "GET /request_log_analyzer.html HTTP/1.1" 200 5669 0.0129
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /request_log_analyzer.html HTTP/1.1" 200 5669 0.0156
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /javascripts/cufon-yui.js HTTP/1.1" 304 - 0.0033
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /javascripts/CartoGothic_Std_400-CartoGothic_Std_700-CartoGothic_Std_italic_400-CartoGothic_Std_italic_700.font.js HTTP/1.1" 304 - 0.0035
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /css/stylesheet.css HTTP/1.1" 304 - 0.0031
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/performance_train.png HTTP/1.1" 304 - 0.0026
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/footer-bg.png HTTP/1.1" 304 - 0.0029
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/nav-bg.png HTTP/1.1" 304 - 0.0048
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/box-bg.png HTTP/1.1" 304 - 0.0055
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/rails_doctors_logo.png HTTP/1.1" 304 - 0.0049
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/radial-bg.png HTTP/1.1" 304 - 0.0029
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/lock.png HTTP/1.1" 304 - 0.0022
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/rla.png HTTP/1.1" 304 - 0.0014
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /images/contact.png HTTP/1.1" 304 - 0.0008
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /favicon.ico HTTP/1.1" 404 420 0.0007
+127.0.0.1 - - [23/Nov/2009 22:46:30] "GET /contact.html HTTP/1.1" 200 4476 0.0149
+127.0.0.1 - - [23/Nov/2009 22:46:31] "GET /contact.html HTTP/1.1" 200 4476 0.0118
+127.0.0.1 - - [23/Nov/2009 22:46:31] "GET /contact.html HTTP/1.1" 200 4476 0.0556
+127.0.0.1 - - [23/Nov/2009 22:46:32] "GET /security.html HTTP/1.1" 200 4503 0.0980
+127.0.0.1 - - [23/Nov/2009 22:46:32] "GET /performance.html HTTP/1.1" 200 4751 0.0109
+127.0.0.1 - - [23/Nov/2009 22:46:33] "GET /index.html HTTP/1.1" 200 5056 0.0169
+127.0.0.1 - - [23/Nov/2009 22:46:33] "GET /blog.html HTTP/1.1" 200 8156 0.0112
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /blog.html HTTP/1.1" 200 8156 0.0113
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /css/stylesheet.css HTTP/1.1" 304 - 0.0017
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /javascripts/cufon-yui.js HTTP/1.1" 304 - 0.0011
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /javascripts/CartoGothic_Std_400-CartoGothic_Std_700-CartoGothic_Std_italic_400-CartoGothic_Std_italic_700.font.js HTTP/1.1" 304 - 0.0007
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/rails_doctors_logo.png HTTP/1.1" 304 - 0.0045
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/performance_train.png HTTP/1.1" 304 - 0.0038
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/footer-bg.png HTTP/1.1" 304 - 0.0044
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/nav-bg.png HTTP/1.1" 304 - 0.0042
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/radial-bg.png HTTP/1.1" 304 - 0.0037
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/box-bg.png HTTP/1.1" 304 - 0.0045
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/rla.png HTTP/1.1" 304 - 0.0026
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/lock.png HTTP/1.1" 304 - 0.0022
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /images/contact.png HTTP/1.1" 304 - 0.0024
+127.0.0.1 - - [23/Nov/2009 22:46:34] "GET /favicon.ico HTTP/1.1" 404 420 0.0007
+127.0.0.1 - - [23/Nov/2009 22:46:36] "GET /blog.html HTTP/1.1" 200 8156 0.0119
+127.0.0.1 - - [23/Nov/2009 22:46:36] "GET /css/stylesheet.css HTTP/1.1" 200 3782 0.0030
+127.0.0.1 - - [23/Nov/2009 22:46:36] "GET /javascripts/cufon-yui.js HTTP/1.1" 200 16944 0.0220
+127.0.0.1 - - [23/Nov/2009 22:46:36] "GET /javascripts/CartoGothic_Std_400-CartoGothic_Std_700-CartoGothic_Std_italic_400-CartoGothic_Std_italic_700.font.js HTTP/1.1" 200 76594 0.0258
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/box-bg.png HTTP/1.1" 200 129 0.0058
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/footer-bg.png HTTP/1.1" 200 267 0.0056
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/nav-bg.png HTTP/1.1" 200 221 0.0052
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/performance_train.png HTTP/1.1" 200 7496 0.0103
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/radial-bg.png HTTP/1.1" 200 9288 0.0079
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/rails_doctors_logo.png HTTP/1.1" 200 11487 0.0106
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/rla.png HTTP/1.1" 200 23857 0.0233
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/contact.png HTTP/1.1" 200 23961 0.0228
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /images/lock.png HTTP/1.1" 200 31780 0.0494
+127.0.0.1 - - [23/Nov/2009 22:46:37] "GET /favicon.ico HTTP/1.1" 404 420 0.0007
+127.0.0.1 - - [23/Nov/2009 22:46:38] "GET /blog.html HTTP/1.1" 200 8156 0.0108
+127.0.0.1 - - [23/Nov/2009 22:46:38] "GET /javascripts/CartoGothic_Std_400-CartoGothic_Std_700-CartoGothic_Std_italic_400-CartoGothic_Std_italic_700.font.js HTTP/1.1" 304 - 0.0017
+127.0.0.1 - - [23/Nov/2009 22:46:38] "GET /javascripts/cufon-yui.js HTTP/1.1" 304 - 0.0013
+127.0.0.1 - - [23/Nov/2009 22:46:38] "GET /css/stylesheet.css HTTP/1.1" 304 - 0.0024
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/rails_doctors_logo.png HTTP/1.1" 304 - 0.0073
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/performance_train.png HTTP/1.1" 304 - 0.0039
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/footer-bg.png HTTP/1.1" 304 - 0.0035
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/radial-bg.png HTTP/1.1" 304 - 0.0054
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/nav-bg.png HTTP/1.1" 304 - 0.0043
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/box-bg.png HTTP/1.1" 304 - 0.0038
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/rla.png HTTP/1.1" 304 - 0.0019
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/lock.png HTTP/1.1" 304 - 0.0014
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /images/contact.png HTTP/1.1" 304 - 0.0012
+127.0.0.1 - - [23/Nov/2009 22:46:39] "GET /favicon.ico HTTP/1.1" 404 420 0.0007
+127.0.0.1 - - [23/Nov/2009 22:46:40] "GET /performance.html HTTP/1.1" 200 4751 0.0110
+127.0.0.1 - - [23/Nov/2009 22:46:40] "GET /performance.html HTTP/1.1" 200 4751 0.0110
+127.0.0.1 - - [23/Nov/2009 22:46:41] "GET /security.html HTTP/1.1" 200 4503 0.0369
+127.0.0.1 - - [23/Nov/2009 22:46:41] "GET /performance.html HTTP/1.1" 200 4751 0.0118
+127.0.0.1 - - [23/Nov/2009 22:46:41] "GET /security.html HTTP/1.1" 200 4503 0.0573
+127.0.0.1 - - [23/Nov/2009 22:46:42] "GET /security.html HTTP/1.1" 200 4503 0.0426
+127.0.0.1 - - [23/Nov/2009 22:46:42] "GET /performance.html HTTP/1.1" 200 4751 0.0111
+127.0.0.1 - - [23/Nov/2009 22:46:43] "GET /security.html HTTP/1.1" 200 4503 0.0108
+127.0.0.1 - - [23/Nov/2009 22:46:43] "GET /performance.html HTTP/1.1" 200 4751 0.0105
+127.0.0.1 - - [23/Nov/2009 22:46:43] "GET /security.html HTTP/1.1" 200 4503 0.0147
+127.0.0.1 - - [23/Nov/2009 22:46:43] "GET /contact.html HTTP/1.1" 200 4476 0.0105
+127.0.0.1 - - [23/Nov/2009 22:46:44] "GET /contact.html HTTP/1.1" 200 4476 0.0110
+127.0.0.1 - - [23/Nov/2009 22:46:44] "GET /contact.html HTTP/1.1" 200 4476 0.0109
+^C
+== Sinatra has ended his set (crowd applauds)

data/spec/unit/file_format/format_autodetection_spec.rb

@@ -7,12 +7,12 @@ describe RequestLogAnalyzer::FileFormat do
       file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:merb))
       file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Merb)
     end
-    
+
     it "should autodetect a MySQL slow query log" do
       file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:mysql_slow_query))
       file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Mysql)
     end
-    
+
     it "should autodetect a Rails 1.x log" do
       file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_1x))
       file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
@@ -27,7 +27,12 @@ describe RequestLogAnalyzer::FileFormat do
       file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:apache_common))
       file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Apache)
     end
-    
+
+    it "should autodetect a Rack access log" do
+      file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:sinatra))
+      file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rack)
+    end
+
     it "should not find any file format with a bogus file" do
       RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:test_order)).should be_nil
     end

data/spec/unit/file_format/rack_format_spec.rb

@@ -0,0 +1,50 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Rack do
+
+  context '"Sinatra" access log parsing' do
+
+    before(:each) do
+      @file_format      = RequestLogAnalyzer::FileFormat.load(:rack)
+      @log_parser       = RequestLogAnalyzer::Source::LogParser.new(@file_format)    
+      @sample = '127.0.0.1 - - [23/Nov/2009 21:47:47] "GET /css/stylesheet.css HTTP/1.1" 200 3782 0.0024'
+    end
+
+    it "should be a valid file format" do
+      @file_format.should be_valid
+    end
+
+    it "should parse access lines and capture all of its fields" do
+      @file_format.should have_line_definition(:access).capturing(:remote_host, :timestamp, :http_method, :path, :http_version, 
+        :http_status, :bytes_sent, :duration)
+    end
+
+    it "should match the sample line" do
+      @file_format.parse_line(@sample).should include(:line_definition, :captures)
+    end
+
+    it "should not match a nonsense line" do
+      @file_format.parse_line('== Sinatra/0.9.4 has taken the stage on 4567 for development with backup from Mongrel').should be_nil
+    end
+
+    it "should parse the sample fields correctly" do
+      match = @file_format.parse_line(@sample)
+      match.should_not be_nil
+    end
+
+    it "should parse and convert the sample fields correctly" do
+      @log_parser.parse_io(StringIO.new(@sample)) do |request|
+        request[:remote_host].should    == '127.0.0.1'
+        request[:timestamp].should      == 20091123214747
+        request[:http_method].should    == 'GET'
+        request[:path].should           == '/css/stylesheet.css'
+        request[:http_version].should   == '1.1'
+        request[:http_status].should    == 200
+        request[:bytes_sent].should     == 3782
+        request[:duration].should       == 0.0024
+      end
+    end
+
+  end
+
+end

data/spec/unit/file_format/rails_format_spec.rb

@@ -54,6 +54,11 @@ describe RequestLogAnalyzer::FileFormat::Rails do
           line = prefix + 'Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]'
           @rails.should parse_line(line).as(:processing).and_capture(:controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET')
         end
+        
+        it "should parse a :processing line correctly when it contains ipv6 address" do
+           line = prefix + 'Processing PeopleController#index (for ::1 at 2008-08-14 21:16:30) [GET]'
+           @rails.should parse_line(line).as(:processing).and_capture(:controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET')
+        end
 
         it "should parse a Rails 2.1 style :completed line correctly" do
           line = prefix + 'Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.5.1
+  version: 1.5.2
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-19 00:00:00 +01:00
+date: 2009-12-02 00:00:00 +01:00
 default_executable: request-log-analyzer
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -33,7 +33,7 @@ dependencies:
       - !ruby/object:Gem::Version 
         version: 1.1.0
     version: 
-description: "    Request log analyzer's purpose is to find ot how your web application is being used and to focus your optimization efforts.\n    This tool will parse all requests in the application's log file and aggregate the information. Once it is finished parsing\n    the log file(s), it will show the requests that take op most server time using various metrics. It can also insert all\n    parsed request information into a database so you can roll your own analysis. It supports Rails- and Merb-based applications\n    and Apache access log files out of the box, but file formats of other applications can easily be supported by supplying an\n    easy to write log file format definition.\n"
+description: "    Request log analyzer's purpose is to find out how your web application is being used, how it performs and to\n    focus your optimization efforts. This tool will parse all requests in the application's log file and aggregate the \n    information. Once it is finished parsing the log file(s), it will show the requests that take op most server time \n    using various metrics. It can also insert all parsed request information into a database so you can roll your own\n    analysis. It supports Rails-, Merb- and Rack-based applications logs, Apache and Amazon S3 access logs and MySQL \n    slow query logs out of the box, but file formats of other applications can easily be supported by supplying an \n    easy to write log file format definition.\n"
 email: 
 - willem@railsdoctors.com
 - bart@railsdoctors.com
@@ -44,8 +44,8 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
-- lib/request_log_analyzer/output/html.rb
 - spec/unit/tracker/hourly_spread_spec.rb
+- lib/request_log_analyzer/output/html.rb
 - DESIGN.rdoc
 - spec/fixtures/rails_unordered.log
 - spec/fixtures/multiple_files_1.log
@@ -60,6 +60,7 @@ files:
 - lib/request_log_analyzer/aggregator.rb
 - spec/fixtures/decompression.log
 - lib/request_log_analyzer/file_format/apache.rb
+- spec/unit/file_format/rack_format_spec.rb
 - spec/fixtures/decompression.log.gz
 - lib/request_log_analyzer/tracker/frequency.rb
 - lib/request_log_analyzer/tracker/duration.rb
@@ -136,6 +137,7 @@ files:
 - spec/unit/tracker/traffic_tracker_spec.rb
 - lib/request_log_analyzer/tracker/timespan.rb
 - spec/unit/controller/log_processor_spec.rb
+- spec/fixtures/sinatra.log
 - spec/unit/filter/field_filter_spec.rb
 - spec/fixtures/merb_prefixed.log
 - lib/request_log_analyzer/log_processor.rb
@@ -194,11 +196,12 @@ rubyforge_project: r-l-a
 rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
-summary: A command line tool to analyze request logs for Apache, Rails, Merb and other web application servers
+summary: A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers
 test_files: 
 - spec/unit/tracker/hourly_spread_spec.rb
 - spec/unit/file_format/amazon_s3_format_spec.rb
 - spec/unit/database/base_class_spec.rb
+- spec/unit/file_format/rack_format_spec.rb
 - spec/unit/filter/filter_spec.rb
 - spec/unit/tracker/count_tracker_spec.rb
 - spec/unit/database/connection_spec.rb