request-log-analyzer 1.12.5 → 1.12.6

data/bin/request-log-analyzer

@@ -76,7 +76,8 @@ rescue CommandLine::Error => e
   puts "Input options:"
   puts "  --after <date>             Only consider requests from <date> or later."
   puts "  --before <date>            Only consider requests before <date>."
-  puts "  --format <format>, -f:     Log file format. amazon_s3, apache, merb, mysql, rack or rails."
+  puts "  --format <format>, -f:     Log file format: amazon_s3, apache, delayed_job, haproxy, merb, mysql,"
+  puts "                             postgresql, rack, rails, rails3, w3c."
   puts "  --reject <field> <value>   Only consider requests where <field> does not match <value>."
   puts "  --select <field> <value>   Only consider requests where <field> matches <value>."
   puts

data/lib/request_log_analyzer.rb

@@ -1,5 +1,5 @@
 require 'date'
-require 'ordered_hash'
+require 'other/ordered_hash'
 
 # RequestLogAnalyzer is the base namespace in which all functionality of RequestLogAnalyzer is implemented.
 # This module itselfs contains some functions to help with class and source file loading. The actual 
@@ -11,7 +11,7 @@ module RequestLogAnalyzer
   
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.12.4"
+  VERSION = "1.12.6"
 
   # Convert a string/symbol in camelcase ({RequestLogAnalyzer::Controller}) to underscores 
   # (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using 

data/lib/request_log_analyzer/file_format.rb

@@ -7,6 +7,7 @@ module RequestLogAnalyzer::FileFormat
   autoload :Rack,             'request_log_analyzer/file_format/rack'
   autoload :Merb,             'request_log_analyzer/file_format/merb'
   autoload :Mysql,            'request_log_analyzer/file_format/mysql'
+  autoload :Nginx,            'request_log_analyzer/file_format/nginx'
   autoload :Postgresql,       'request_log_analyzer/file_format/postgresql'
   autoload :DelayedJob,       'request_log_analyzer/file_format/delayed_job'
   autoload :DelayedJob2,      'request_log_analyzer/file_format/delayed_job2'

data/lib/request_log_analyzer/file_format/apache.rb

@@ -20,6 +20,7 @@ module RequestLogAnalyzer::FileFormat
     LOG_FORMAT_DEFAULTS = {
       :common   => '%h %l %u %t "%r" %>s %b',
       :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
+      :nginx    => '%a %t %h %u "%r" %>s %b',
       :rack     => '%h %l %u %t "%r" %>s %b %T',
       :referer  => '%{Referer}i -> %U',
       :agent    => '%{User-agent}i'

data/lib/request_log_analyzer/file_format/nginx.rb

@@ -0,0 +1,9 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Nginx < Apache
+
+    def self.create(*args)
+      super(:nginx, *args)
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -61,13 +61,21 @@ module RequestLogAnalyzer::FileFormat
       line.capture(:line).as(:integer)
       line.capture(:file)
     end
-    
+
+    # Rendered queries/index.html.erb (0.6ms)
+    line_definition :rendered do |line|
+      line.compound = [:partial_duration]
+      line.teaser = / Rendered /
+      line.regexp = / Rendered ([a-zA-Z0-9_\-\/.]+(?:\/[a-zA-Z0-9_\-.]+)+)(?:\ within\ .*?)? \((\d+(?:\.\d+)?)ms\)/
+      line.capture(:rendered_file)
+      line.capture(:partial_duration).as(:duration, :unit => :msec)
+    end
+
     # # Not parsed at the moment:
     # SQL (0.2ms) SET SQL_AUTO_IS_NULL=0
     # Query Load (0.4ms) SELECT `queries`.* FROM `queries`
     # Rendered collection (0.0ms)
-    # Rendered queries/index.html.erb (0.6ms)
-    
+
     REQUEST_CATEGORIZER = lambda { |request| "#{request[:controller]}##{request[:action]}.#{request[:format]}" }
     
     report do |analyze|
@@ -80,6 +88,7 @@ module RequestLogAnalyzer::FileFormat
       analyze.frequency :status, :title => 'HTTP statuses returned'
       
       analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration", :line_type => :completed
+      analyze.duration :partial_duration, :category => :rendered_file, :title => 'Partials rendering time', :line_type => :rendered
       analyze.duration :view, :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
       analyze.duration :db, :category => REQUEST_CATEGORIZER, :title => "Database time", :line_type => :completed
       

data/lib/request_log_analyzer/line_definition.rb

@@ -44,7 +44,7 @@ module RequestLogAnalyzer
     end
 
     attr_reader :name
-    attr_accessor :teaser, :regexp, :captures
+    attr_accessor :teaser, :regexp, :captures, :compound
     attr_accessor :header, :footer
 
     alias_method :header?, :header
@@ -56,6 +56,7 @@ module RequestLogAnalyzer
       @name     = name
       @captures = []
       @teaser   = nil
+      @compound = []
       definition.each { |key, value| self.send("#{key.to_s}=".to_sym, value) }
     end
 

data/lib/request_log_analyzer/request.rb

@@ -116,6 +116,7 @@ module RequestLogAnalyzer
       value_hash[:line_type] = parsed_line[:line_definition].name
       value_hash[:lineno] = parsed_line[:lineno]
       value_hash[:source] = parsed_line[:source]
+      value_hash[:compound] = parsed_line[:line_definition].compound
       add_line_hash(value_hash)
     end
 
@@ -124,7 +125,19 @@ module RequestLogAnalyzer
     # The line should be provides as a hash of the fields parsed from the line.
     def add_line_hash(value_hash)
       @lines << value_hash
-      @attributes = value_hash.merge(@attributes)
+      if value_hash[:compound]
+        value_hash.each do |key, value|
+          if value_hash[:compound].include?(key)
+            @attributes[key] = [] if @attributes[key].nil?
+            @attributes[key] = [@attributes[key]] unless @attributes[key].is_a?(Array)
+            @attributes[key] << value
+          else
+            @attributes[key] = value unless key == :compound || @attributes[key]
+          end
+        end
+      else
+        @attributes = value_hash.merge(@attributes)
+      end
     end
 
     # Adds another line to the request. This method switches automatically between

data/lib/request_log_analyzer/source/log_parser.rb

@@ -118,6 +118,11 @@ module RequestLogAnalyzer::Source
     # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
     def parse_file(file, options = {}, &block)
 
+      if File.directory?(file)
+        parse_files(Dir["#{ file }/*"], options, &block)
+        return
+      end
+
       @current_source = File.expand_path(file)
       @source_changes_handler.call(:started, @current_source) if @source_changes_handler
 

data/lib/request_log_analyzer/tracker/numeric_value.rb

@@ -49,11 +49,10 @@ module RequestLogAnalyzer::Tracker
         found_categories.each_with_index do |cat, index|
           update_statistics(cat, found_values[index]) if cat && found_values[index].kind_of?(Numeric)
         end
-
       else
         category = @categorizer.call(request)
         value    = @valueizer.call(request)
-        update_statistics(category, value) if value.kind_of?(Numeric) && category
+        update_statistics(category, value) if (value.kind_of?(Numeric) || value.kind_of?(Array)) && category
       end
     end
 
@@ -227,6 +226,8 @@ module RequestLogAnalyzer::Tracker
     # <tt>category</tt>:: The category for which to update the running statistics calculations
     # <tt>number</tt>:: The numeric value to update the calculations with.
     def update_statistics(category, number)
+      return number.map {|n| update_statistics(category, n)} if number.is_a?(Array)
+
       @categories[category] ||= { :hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number, 
                                   :buckets => Array.new(@number_of_buckets, 0) }
       

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.12.5"
-  s.date    = "2012-09-20"
+  s.version = "1.12.6"
+  s.date    = "2012-12-10"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -45,6 +45,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(.gitignore .infinity_test .travis.yml DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/ordered_hash.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_unordered.log spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
+  s.files      = %w(.gitignore .infinity_test .travis.yml DESIGN.rdoc Gemfile LICENSE README.rdoc Rakefile bin/request-log-analyzer lib/cli/command_line_arguments.rb lib/cli/database_console.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/cli/tools.rb lib/other/ordered_hash.rb lib/request_log_analyzer.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/echo.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/controller.rb lib/request_log_analyzer/database.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/database/warning.rb lib/request_log_analyzer/file_format.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/delayed_job2.rb lib/request_log_analyzer/file_format/delayed_job21.rb lib/request_log_analyzer/file_format/haproxy.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/mysql.rb lib/request_log_analyzer/file_format/nginx.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb lib/request_log_analyzer/file_format/rails3.rb lib/request_log_analyzer/file_format/rails_development.rb lib/request_log_analyzer/file_format/w3c.rb lib/request_log_analyzer/filter.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/mailer.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/hourly_spread.rb lib/request_log_analyzer/tracker/numeric_value.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/tracker/traffic.rb request-log-analyzer.gemspec spec/database.yml spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/decompression.log spec/fixtures/decompression.log.bz2 spec/fixtures/decompression.log.gz spec/fixtures/decompression.log.zip spec/fixtures/decompression.tar.gz spec/fixtures/decompression.tgz spec/fixtures/header_and_footer.log spec/fixtures/merb.log spec/fixtures/merb_prefixed.log spec/fixtures/multiple_files_1.log spec/fixtures/multiple_files_2.log spec/fixtures/mysql_slow_query.log spec/fixtures/oink_22.log spec/fixtures/oink_22_failure.log spec/fixtures/postgresql.log spec/fixtures/rails.db spec/fixtures/rails_1x.log spec/fixtures/rails_22.log spec/fixtures/rails_22_cached.log spec/fixtures/rails_3_partials.log spec/fixtures/rails_unordered.log spec/fixtures/s3_logs/2012-10-05-16-18-11-F9AAC5D1A55AEBAD spec/fixtures/s3_logs/2012-10-05-16-26-06-15314AF7F0651839 spec/fixtures/sinatra.log spec/fixtures/syslog_1x.log spec/fixtures/test_file_format.log spec/fixtures/test_language_combined.log spec/fixtures/test_order.log spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/lib/helpers.rb spec/lib/macros.rb spec/lib/matchers.rb spec/lib/mocks.rb spec/lib/testing_format.rb spec/spec_helper.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb tasks/github-gem.rake tasks/request_log_analyzer.rake)
   s.test_files = %w(spec/integration/command_line_usage_spec.rb spec/integration/mailer_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/integration/scout_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/controller/controller_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/database/base_class_spec.rb spec/unit/database/connection_spec.rb spec/unit/database/database_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/delayed_job21_format_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/haproxy_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/file_format/rails_format_spec.rb spec/unit/file_format/w3c_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/mailer_spec.rb spec/unit/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/traffic_tracker_spec.rb)
 end

data/spec/fixtures/rails_3_partials.log

@@ -0,0 +1,13 @@
+Started GET "/foobartest" for 127.0.0.1 at 2012-11-21 15:21:13 +0100
+Processing by HomeController#foobartest as */*
+  Rendered home/foobartest_partial.html.slim (10.0ms)
+  Rendered home/foobartest_partial.html.slim (10.0ms)
+  Rendered home/foobartest_partial.html.slim (10.0ms)
+Completed 200 OK in 58ms (Views: 44.6ms | ActiveRecord: 0.0ms)
+
+
+Started GET "/foobartest" for 127.0.0.1 at 2012-11-21 15:21:31 +0100
+Processing by HomeController#foobartest as */*
+  Rendered home/foobartest_partial.html.slim (100.0ms)
+  Rendered home/foobartest_partial.html.slim (0.0ms)
+Completed 200 OK in 2ms (Views: 0.6ms | ActiveRecord: 0.0ms)

data/spec/fixtures/s3_logs/2012-10-05-16-18-11-F9AAC5D1A55AEBAD

@@ -0,0 +1,5 @@
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:24:46 +0000] 174.44.160.70 - CE74FF983317B326 REST.GET.OBJECT public/portfolio/22/thumbnail.jpg "GET /lamestuff.com/public/portfolio/22/thumbnail.jpg HTTP/1.1" 200 - 9515 9515 42 41 "http://spike.grobste.in/portfolio" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:20 +0000] 174.44.160.70 - D484E1F8E6DE0AAB REST.GET.OBJECT public/projects/46/resume.png "GET /lamestuff.com/public/projects/46/resume.png HTTP/1.1" 200 - 510856 510856 85 74 "http://spike.grobste.in/portfolio/show?name=lamestuff.com-%282007%29" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:25 +0000] 174.44.160.70 - E5C48446A01539FC REST.GET.OBJECT public/projects/3/gameboy.jpg "GET /lamestuff.com/public/projects/3/gameboy.jpg HTTP/1.1" 200 - 37458 37458 57 55 "http://spike.grobste.in/project/classic-console-art" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:27 +0000] 174.44.160.70 - CD6CC6713DE2254A REST.GET.OBJECT public/projects/115/xmas-card-2006-d.jpg "GET /lamestuff.com/public/projects/115/xmas-card-2006-d.jpg HTTP/1.1" 200 - 89892 89892 58 55 "http://spike.grobste.in/portfolio/show?name=xmas-2006" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:31 +0000] 174.44.160.70 - B8B9CBDE2EFE51F3 REST.GET.OBJECT public/projects/34/Tim_Horton-Window-Coffee-ba.jpg "GET /lamestuff.com/public/projects/34/Tim_Horton-Window-Coffee-ba.jpg HTTP/1.1" 200 - 27376 27376 41 40 "http://spike.grobste.in/project/tim-horton's-at-the-nyse" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -

data/spec/fixtures/s3_logs/2012-10-05-16-26-06-15314AF7F0651839

@@ -0,0 +1,3 @@
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:24:46 +0000] 174.44.160.70 - FC6B8CCFF3510D92 REST.GET.OBJECT public/portfolio/29/thumbnail.jpg "GET /lamestuff.com/public/portfolio/29/thumbnail.jpg HTTP/1.1" 200 - 6240 6240 40 39 "http://spike.grobste.in/portfolio" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:27 +0000] 174.44.160.70 - 6908D7E8249F8AB0 REST.GET.OBJECT public/projects/114/xmas-card-2006-c.jpg "GET /lamestuff.com/public/projects/114/xmas-card-2006-c.jpg HTTP/1.1" 200 - 99874 99874 51 47 "http://spike.grobste.in/portfolio/show?name=xmas-2006" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -
+300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:30 +0000] 174.44.160.70 - 41804BB9A626C674 REST.GET.OBJECT public/projects/28/Tim_Horton_Counter__3x12_.jpg "GET /lamestuff.com/public/projects/28/Tim_Horton_Counter__3x12_.jpg HTTP/1.1" 200 - 22256 22256 93 91 "http://spike.grobste.in/project/tim-horton's-at-the-nyse" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" -

data/spec/integration/command_line_usage_spec.rb

@@ -20,7 +20,6 @@ describe RequestLogAnalyzer, 'running from command line' do
     output.any? { |line| /^Parsed requests\:\s*2\s/ =~ line }.should be_true
   end
 
-
   it "should skip 1 requests with a --select option" do
     output = run("#{log_fixture(:rails_1x)} --select controller PeopleController")
     output.any? { |line| /^Skipped requests\:\s*1\s/ =~ line }.should be_true
@@ -87,4 +86,9 @@ describe RequestLogAnalyzer, 'running from command line' do
     output = run("--format rails - < #{log_fixture(:rails_1x)}")
     output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
   end
+
+  it "should accept a directory as a commandline option" do
+    output = run("#{log_directory_fixture("s3_logs")} --format amazon_s3")
+    output.any? { |line| /^Parsed requests:\s*8\s/ =~ line }.should be_true
+  end
 end

data/spec/lib/helpers.rb

@@ -10,6 +10,11 @@ module RequestLogAnalyzer::RSpec::Helpers
     File.dirname(__FILE__) + "/../fixtures/#{name}.#{extention}"
   end
 
+  # directory of logs
+  def log_directory_fixture(name)
+    File.dirname(__FILE__) + "/../fixtures/#{name}"
+  end
+
   # Creates a log file given some lines
   def log_snippet(*lines)
     StringIO.new(lines.join("\n") << "\n")
@@ -49,12 +54,12 @@ module RequestLogAnalyzer::RSpec::Helpers
   def temp_output_file(file_type)
     File.expand_path("#{File.dirname(__FILE__)}/../../tmp/spec.#{file_type}.tmp")
   end
-  
+
   # Check if a given string can be found in the given file
   # Returns the line number if found, nil otherwise
   def find_string_in_file(string, file, options = {})
     return nil unless File.exists?(file)
-    
+
     line_counter = 0
 
     File.open( file ) do |io|
@@ -66,7 +71,7 @@ module RequestLogAnalyzer::RSpec::Helpers
         return line_counter if line.include? string
       }
     end
-    
+
     return nil
   end
 end

data/spec/unit/file_format/rails3_format_spec.rb

@@ -5,7 +5,7 @@ describe RequestLogAnalyzer::FileFormat::Rails3 do
   subject { RequestLogAnalyzer::FileFormat.load(:rails3) }
   
   it { should be_well_formed }
-  it { should have(9).report_trackers }
+  it { should have(10).report_trackers }
   
   describe '#parse_line' do
 
@@ -81,6 +81,11 @@ describe RequestLogAnalyzer::FileFormat::Rails3 do
           :message => "undefined local variable or method `field' for #<Class>", 
           :file    => '/Users/willem/Code/warehouse/app/views/queries/execute.csv.erb')
     end
+
+    it "should parse :rendered lines as an array" do
+      line = " Rendered queries/index.html.erb (0.6ms)"
+      subject.should parse_line(line).as(:rendered).and_capture(:partial_duration => [0.0006])
+    end
   end
   
   describe '#parse_io' do
@@ -100,7 +105,19 @@ describe RequestLogAnalyzer::FileFormat::Rails3 do
       log_parser.should_not_receive(:warn)
       log_parser.parse_string(log)
     end
-    
+
+    it "should count partials correctly" do
+      log = <<-EOLOG
+        Started GET "/stream_support" for 127.0.0.1 at 2012-11-21 15:21:31 +0100
+        Processing by HomeController#stream_support as */*
+          Rendered home/stream_support.html.slim (33.2ms)
+          Rendered home/stream_support.html.slim (0.0ms)
+        Completed 200 OK in 2ms (Views: 0.6ms | ActiveRecord: 0.0ms)
+      EOLOG
+
+      log_parser.parse_string(log)
+    end
+
     it "should parse an unroutable request correctly" do
       log = <<-EOLOG
         Started GET "/404" for 127.0.0.1 at Fri Mar 19 06:40:57 -0700 2010
@@ -113,10 +130,10 @@ describe RequestLogAnalyzer::FileFormat::Rails3 do
       EOLOG
 
       log_parser.should_receive(:handle_request).once
-      log_parser.should_not_receive(:warn)
+      log_parser.should_receive(:warn).once
       log_parser.parse_string(log)
     end
-    
+
     it "should parse a failing request correctly" do
       log = <<-EOLOG
         Started POST "/queries/397638749/execute.csv" for 127.0.0.1 at Mon Mar 01 18:44:33 -0800 2010
@@ -142,7 +159,7 @@ describe RequestLogAnalyzer::FileFormat::Rails3 do
       EOLOG
 
       log_parser.should_receive(:handle_request).once
-      log_parser.should_not_receive(:warn)
+      log_parser.should_receive(:warn).exactly(3).times
       log_parser.parse_string(log)
     end
   end

data/spec/unit/tracker/duration_tracker_spec.rb

@@ -24,6 +24,13 @@ describe RequestLogAnalyzer::Tracker::Duration do
       lambda { @tracker.report(mock_output) }.should_not raise_error
     end
 
+    it "should generate a report with arrays of durations are present" do
+      @tracker.update(request(:category => 'a', :duration => [0.1, 0.2]))
+      @tracker.update(request(:category => 'a', :duration => [0.2, 0.3]))
+      lambda { @tracker.report(mock_output) }.should_not raise_error
+      @tracker.to_yaml_object['a'].should include(:min => 0.1, :hits => 4, :max => 0.3, :mean => 0.2, :sum => 0.8)
+    end
+
     it "should generate a YAML output" do
       @tracker.update(request(:category => 'a', :duration => 0.2))
       @tracker.update(request(:category => 'b', :duration => 0.2))

metadata

@@ -1,83 +1,100 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: request-log-analyzer
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 1
-  - 12
-  - 5
-  version: 1.12.5
+version: !ruby/object:Gem::Version
+  version: 1.12.6
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Willem van Bergen
 - Bart ten Brinke
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-09-20 00:00:00 -04:00
-default_executable: request-log-analyzer
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-12-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: rake
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: rspec
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    requirements: 
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 2
-        - 8
-        version: "2.8"
+      - !ruby/object:Gem::Version
+        version: '2.8'
   type: :development
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: activerecord
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: sqlite3
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        version: "0"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :development
-  version_requirements: *id004
-description: "    Request log analyzer's purpose is to find out how your web application is being used, how it performs and to\n    focus your optimization efforts. This tool will parse all requests in the application's log file and aggregate the \n    information. Once it is finished parsing the log file(s), it will show the requests that take op most server time \n    using various metrics. It can also insert all parsed request information into a database so you can roll your own\n    analysis. It supports Rails-, Merb- and Rack-based applications logs, Apache and Amazon S3 access logs and MySQL \n    slow query logs out of the box, but file formats of other applications can easily be supported by supplying an \n    easy to write log file format definition.\n"
-email: 
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "    Request log analyzer's purpose is to find out how your web application
+  is being used, how it performs and to\n    focus your optimization efforts. This
+  tool will parse all requests in the application's log file and aggregate the \n
+  \   information. Once it is finished parsing the log file(s), it will show the requests
+  that take op most server time \n    using various metrics. It can also insert all
+  parsed request information into a database so you can roll your own\n    analysis.
+  It supports Rails-, Merb- and Rack-based applications logs, Apache and Amazon S3
+  access logs and MySQL \n    slow query logs out of the box, but file formats of
+  other applications can easily be supported by supplying an \n    easy to write log
+  file format definition.\n"
+email:
 - willem@railsdoctors.com
 - bart@railsdoctors.com
-executables: 
+executables:
 - request-log-analyzer
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.rdoc
-files: 
+files:
 - .gitignore
 - .infinity_test
 - .travis.yml
@@ -92,7 +109,7 @@ files:
 - lib/cli/database_console_init.rb
 - lib/cli/progressbar.rb
 - lib/cli/tools.rb
-- lib/ordered_hash.rb
+- lib/other/ordered_hash.rb
 - lib/request_log_analyzer.rb
 - lib/request_log_analyzer/aggregator.rb
 - lib/request_log_analyzer/aggregator/database_inserter.rb
@@ -114,6 +131,7 @@ files:
 - lib/request_log_analyzer/file_format/haproxy.rb
 - lib/request_log_analyzer/file_format/merb.rb
 - lib/request_log_analyzer/file_format/mysql.rb
+- lib/request_log_analyzer/file_format/nginx.rb
 - lib/request_log_analyzer/file_format/oink.rb
 - lib/request_log_analyzer/file_format/postgresql.rb
 - lib/request_log_analyzer/file_format/rack.rb
@@ -164,7 +182,10 @@ files:
 - spec/fixtures/rails_1x.log
 - spec/fixtures/rails_22.log
 - spec/fixtures/rails_22_cached.log
+- spec/fixtures/rails_3_partials.log
 - spec/fixtures/rails_unordered.log
+- spec/fixtures/s3_logs/2012-10-05-16-18-11-F9AAC5D1A55AEBAD
+- spec/fixtures/s3_logs/2012-10-05-16-26-06-15314AF7F0651839
 - spec/fixtures/sinatra.log
 - spec/fixtures/syslog_1x.log
 - spec/fixtures/test_file_format.log
@@ -221,42 +242,40 @@ files:
 - spec/unit/tracker/traffic_tracker_spec.rb
 - tasks/github-gem.rake
 - tasks/request_log_analyzer.rake
-has_rdoc: true
 homepage: http://railsdoctors.com
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --title
 - request-log-analyzer
 - --main
 - README.rdoc
 - --line-numbers
 - --inline-source
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-requirements: 
-- To use the database inserter, ActiveRecord and an appropriate database adapter are required.
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- To use the database inserter, ActiveRecord and an appropriate database adapter are
+  required.
 rubyforge_project: r-l-a
-rubygems_version: 1.3.6
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
-summary: A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers
-test_files: 
+summary: A command line tool to analyze request logs for Apache, Rails, Merb, MySQL
+  and other web application servers
+test_files:
 - spec/integration/command_line_usage_spec.rb
 - spec/integration/mailer_spec.rb
 - spec/integration/munin_plugins_rails_spec.rb