request-log-analyzer 1.6.2 → 1.6.3

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
 
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.6.2"
+  VERSION = "1.6.3"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -11,24 +11,25 @@ module RequestLogAnalyzer::FileFormat
     line_definition :access do |line|
       line.header = true
       line.footer = true
-      line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
-      line.captures << { :name => :bucket_owner,    :type => :string } <<
-                       { :name => :bucket,          :type => :string } <<
-                       { :name => :timestamp,       :type => :timestamp } <<
-                       { :name => :remote_ip,       :type => :string } <<
-                       { :name => :requester,       :type => :string } <<
-                       { :name => :request_id,      :type => :string } <<
-                       { :name => :operation,       :type => :string } <<
-                       { :name => :key,             :type => :nillable_string } <<
-                       { :name => :request_uri,     :type => :string } <<
-                       { :name => :http_status,     :type => :integer } <<
-                       { :name => :error_code,      :type => :nillable_string } <<
-                       { :name => :bytes_sent,      :type => :traffic,  :unit => :byte } <<
-                       { :name => :object_size,     :type => :traffic,  :unit => :byte } <<
-                       { :name => :total_time,      :type => :duration, :unit => :msec } <<
-                       { :name => :turnaround_time, :type => :duration, :unit => :msec } <<
-                       { :name => :referer,         :type => :referer } <<
-                       { :name => :user_agent,      :type => :user_agent }
+      line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]*)" "([^"]*)"/
+      
+      line.capture(:bucket_owner)
+      line.capture(:bucket)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:remote_ip)
+      line.capture(:requester)
+      line.capture(:request_id)
+      line.capture(:operation)
+      line.capture(:key).as(:nillable_string)
+      line.capture(:request_uri)
+      line.capture(:http_status).as(:integer)
+      line.capture(:error_code).as(:nillable_string)
+      line.capture(:bytes_sent).as(:traffic, :unit => :byte)
+      line.capture(:object_size).as(:traffic, :unit => :byte)
+      line.capture(:total_time).as(:duration, :unit => :msec)
+      line.capture(:turnaround_time).as(:duration, :unit => :msec)
+      line.capture(:referer).as(:referer)
+      line.capture(:user_agent).as(:user_agent)
     end
 
     report do |analyze|

data/lib/request_log_analyzer/file_format/delayed_job.rb

@@ -7,38 +7,42 @@ module RequestLogAnalyzer::FileFormat
     line_definition :job_lock do |line|
       line.header = true
       line.regexp = /\* \[JOB\] acquiring lock on (\S+)/
-      line.captures << { :name => :job, :type => :string }
+      
+      line.capture(:job)
     end
     
     line_definition :job_completed do |line|
       line.footer = true
       line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/
-      line.captures << { :name => :completed_job, :type => :string } << 
-                       { :name => :duration, :type => :duration, :unit => :sec }
+
+      line.capture(:completed_job)
+      line.capture(:duration).as(:duration, :unit => :sec) 
     end
     
     line_definition :job_failed do |line|
       line.footer = true
       line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/
-      line.captures << { :name => :failed_job, :type => :string  } << 
-                       { :name => :exception,  :type => :string  } << 
-                       { :name => :attempts,   :type => :integer }
       
+      line.capture(:failed_job)
+      line.capture(:exception)
+      line.capture(:attempts).as(:integer)
     end
     
     line_definition :job_lock_failed do |line|
       line.footer = true
       line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/
-      line.captures << { :name => :locked_job, :type => :string }
+      
+      line.capture(:locked_job)
     end
     
     # line_definition :batch_completed do |line|
     #   line.header = true
     #   line.footer = true
     #   line.regexp = /(\d+) jobs processed at (\d+\.\d+) j\/s, (\d+) failed .../
-    #   line.captures << { :name => :total_amount,  :type => :integer } << 
-    #                    { :name => :mean_duration, :type => :duration, :unit => :sec } <<
-    #                    { :name => :failed_amount, :type => :integer }
+    #
+    #   line.capture(:total_amount).as(:integer)
+    #   line.capture(:mean_duration).as(:duration, :unit => :sec)
+    #   line.capture(:failed_amount).as(:integer)
     # end
     
     report do |analyze|

data/lib/request_log_analyzer/file_format/mysql.rb

@@ -8,42 +8,45 @@ module RequestLogAnalyzer::FileFormat
       line.header = :alternative
       line.teaser = /\# Time: /
       line.regexp = /\# Time: (#{timestamp('%y%m%d %k:%M:%S')})/
-      line.captures << { :name => :timestamp, :type => :timestamp }
+      
+      line.capture(:timestamp).as(:timestamp)
     end
 
     line_definition :user_host do |line|
       line.header = :alternative
       line.teaser = /\# User\@Host\: /
       line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[(#{ip_address(true)})\]/
-      line.captures << { :name => :user, :type => :string } << 
-                       { :name => :host, :type => :string } <<
-                       { :name => :ip,   :type => :string }
+      
+      line.capture(:user)
+      line.capture(:host)
+      line.capture(:ip)
     end
 
     line_definition :query_statistics do |line|
       line.header = :alternative
       line.teaser = /\# Query_time: /
       line.regexp = /\# Query_time: (\d+(?:\.\d+)?)\s+Lock_time: (\d+(?:\.\d+)?)\s+Rows_sent: (\d+)\s+Rows_examined: (\d+)/
-      line.captures << { :name => :query_time, :type => :duration, :unit => :sec } <<
-                       { :name => :lock_time,  :type => :duration, :unit => :sec } <<
-                       { :name => :rows_sent, :type => :integer } <<
-                       { :name => :rows_examined, :type => :integer }
+      
+      line.capture(:query_time).as(:duration, :unit => :sec)
+      line.capture(:lock_time).as(:duration, :unit => :sec)
+      line.capture(:rows_sent).as(:integer)
+      line.capture(:rows_examined).as(:integer)
     end
 
     line_definition :use_database do |line|
       line.regexp   = /^use (\w+);\s*$/
-      line.captures << { :name => :database, :type => :string }
+      line.capture(:database)
     end
 
     line_definition :query_part do |line|
       line.regexp   = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
-      line.captures << { :name => :query_fragment, :type => :string }
+      line.capture(:query_fragment)
     end
 
     line_definition :query do |line|
-      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
-      line.captures << { :name => :query, :type => :sql }
       line.footer = true
+      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
+      line.capture(:query).as(:sql)
     end
 
     PER_USER       = :user

data/lib/request_log_analyzer/file_format/postgresql.rb

@@ -8,21 +8,23 @@ module RequestLogAnalyzer::FileFormat
       line.header = true
       line.teaser = /LOG\:  query\:/
       line.regexp = /(#{timestamp('%y-%m-%d %k:%M:%S')})\ LOG:  query:\s+(.*)/
-      line.captures << { :name => :timestamp, :type => :timestamp } <<
-                       { :name => :query_fragment, :type => :string }
+
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:query_fragment)
     end
       
     line_definition :duration do |line|
       line.footer = true
       line.teaser = /duration:/
       line.regexp = /#{timestamp('%y-%m-%d %k:%M:%S')}\ LOG\:  duration\: (.*)(\ )sec/
-      line.captures << { :name => :query_time, :type => :duration, :unit => :sec } <<
-                       { :name => :query, :type => :sql } # Hack to gather up fragments
+
+      line.capture(:query_time).as(:duration, :unit => :sec)
+      line.capture(:query).as(:sql) # Hack to gather up fragments
     end
     
     line_definition :query_fragment do |line|
       line.regexp = /^(?!.*LOG)\s*(.*)\s*/
-      line.captures << { :name => :query_fragment, :type => :string }
+      line.capture(:query_fragment)
     end
 
     report do |analyze|

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -0,0 +1,86 @@
+module RequestLogAnalyzer::FileFormat
+
+  # Default FileFormat class for Rails 3 logs.
+  #
+  # For now, this is just a basic implementation. It will probaby change after 
+  # Rails 3 final has been released.
+  class Rails3 < Base
+
+    extend CommonRegularExpressions
+
+    # Started GET "/queries" for 127.0.0.1 at 2010-02-25 16:15:18
+    line_definition :started do |line|
+      line.header = true
+      line.teaser = /Started /
+      line.regexp = /Started ([A-Z]+) "([^"]+)" for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})/
+      
+      line.capture(:method)
+      line.capture(:path)
+      line.capture(:ip)
+      line.capture(:timestamp).as(:timestamp)
+    end
+    
+    # Processing by QueriesController#index as HTML
+    line_definition :processing do |line|
+      line.teaser = /Processing by /
+      line.regexp = /Processing by (\w+)\#(\w+) as (\w+)/
+      
+      line.capture(:controller)
+      line.capture(:action)
+      line.capture(:format)
+    end
+    
+    # Completed in 9ms (Views: 4.9ms | ActiveRecord: 0.5ms) with 200
+    line_definition :completed do |line|
+      line.footer = true
+      line.teaser = /Completed /
+      line.regexp = /Completed (\d+) .* in (\d+)ms \([^\)]*\)/
+
+      line.capture(:status).as(:integer)
+      line.capture(:duration).as(:duration, :unit => :msec)
+    end
+    
+    # ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:
+    line_definition :failure do |line|
+      line.footer = true
+      line.regexp = /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:\s*$/
+
+      line.capture(:error)
+      line.capture(:message)
+      line.capture(:line).as(:integer)
+      line.capture(:file)
+    end
+    
+    # # Not parsed at the moment:
+    #  SQL (0.2ms)  SET SQL_AUTO_IS_NULL=0
+    #  Query Load (0.4ms)  SELECT `queries`.* FROM `queries`
+    # Rendered collection (0.0ms)
+    # Rendered queries/index.html.erb (0.6ms)
+    
+    REQUEST_CATEGORIZER = lambda { |request| "#{request[:controller]}##{request[:action]}.#{request[:format]}" }
+    
+    report do |analyze|
+      
+      analyze.timespan
+      analyze.hourly_spread
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Most requested'
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      
+      analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
+      # analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+      # analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
+        :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
+    end
+    
+    class Request < RequestLogAnalyzer::Request
+      def convert_timestamp(value, defintion)
+        value.gsub(/[^0-9]/, '')[0...14].to_i
+      end
+    end
+
+  end
+end

data/lib/request_log_analyzer/line_definition.rb

@@ -26,6 +26,19 @@ module RequestLogAnalyzer
       end
     end
 
+    class CaptureDefiner
+      attr_accessor :capture_hash
+      
+      def initialize(hash)
+        @capture_hash = hash
+      end
+      
+      def as(type, type_options = {})
+        @capture_hash.merge!(type_options.merge(:type => type))
+        return self
+      end
+    end
+
     attr_reader :name
     attr_accessor :teaser, :regexp, :captures
     attr_accessor :header, :footer
@@ -47,6 +60,12 @@ module RequestLogAnalyzer
       yield(definition) if block_given?
       return definition
     end
+    
+    def capture(name)
+      new_capture_hash = { :name => name, :type => :string}
+      captures << new_capture_hash
+      CaptureDefiner.new(new_capture_hash)
+    end
 
     # Checks whether a given line matches this definition.
     # It will return false if a line does not match. If the line matches, a hash is returned

data/lib/request_log_analyzer/output/fancy_html.rb

@@ -25,7 +25,25 @@ module RequestLogAnalyzer::Output
     def report_hourly_spread(tracker)
       title tracker.title
       puts tag(:img, nil, :width => '700', :height => '120', :src =>
-          Gchart.sparkline(:data => tracker.hour_frequencies, :size => '700x120', :line_colors => '0077CC'))
+          Gchart.sparkline(:data => tracker.hour_frequencies, :size => '700x120', :line_colors => '0077CC',
+                           :axis_with_labels => 'x,y', :axis_labels => [x_axis_labels(tracker),y_axis_labels(tracker)]))
+    end
+    
+    def x_axis_labels(tracker)
+      frmt = '%H:%M'
+      x_axis_labels = []
+      num_labels    = 5
+      start         = tracker.first_timestamp
+      step          = tracker.timespan / (num_labels - 1)
+      for i in 0...num_labels do
+        x_axis_labels << (start + step * i).strftime(frmt)
+      end
+      x_axis_labels
+    end
+    
+    def y_axis_labels(tracker)
+      sorted_frequencies = tracker.hour_frequencies.sort
+      [sorted_frequencies.first, sorted_frequencies.last]
     end
   end
 end

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.6.2"
-  s.date    = "2010-01-29"
+  s.version = "1.6.3"
+  s.date    = "2010-03-19"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -36,6 +36,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(spec/unit/tracker/hourly_spread_spec.rb lib/request_log_analyzer/output/html.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/unit/file_format/rack_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log spec/unit/filter/timespan_filter_spec.rb lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/postgresql.log spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/integration/munin_plugins_rails_spec.rb spec/lib/helpers.rb spec/unit/file_format/delayed_job_format_spec.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/tracker/numeric_value.rb spec/fixtures/test_language_combined.log spec/unit/request_spec.rb lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/output/fancy_html.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log spec/unit/mailer_spec.rb lib/request_log_analyzer/file_format/rails.rb Rakefile lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/fixtures/sinatra.log spec/unit/filter/field_filter_spec.rb spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
-  s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/filter/filter_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
+  s.files      = %w(spec/unit/tracker/hourly_spread_spec.rb lib/request_log_analyzer/output/html.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/unit/file_format/rack_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake spec/unit/filter/filter_spec.rb .gitignore spec/lib/macros.rb spec/spec_helper.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log spec/unit/filter/timespan_filter_spec.rb lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/postgresql.log spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/unit/file_format/rails3_format_spec.rb spec/integration/munin_plugins_rails_spec.rb lib/request_log_analyzer/file_format/rails3.rb spec/lib/helpers.rb spec/unit/file_format/delayed_job_format_spec.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/tracker/numeric_value.rb spec/fixtures/test_language_combined.log spec/unit/request_spec.rb lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/output/fancy_html.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log spec/unit/mailer_spec.rb lib/request_log_analyzer/file_format/rails.rb Rakefile lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/fixtures/sinatra.log spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
+  s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/filter/filter_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
 end

data/spec/unit/file_format/amazon_s3_format_spec.rb

@@ -46,4 +46,22 @@ describe RequestLogAnalyzer::FileFormat::AmazonS3 do
     end
   end
 
+  it "should parse a COPY request correctly" do
+    line = '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a testbucket [03/Mar/2010:23:04:59 +0000] 174.119.31.76 09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a ACCC34B843C87BC9 REST.COPY.OBJECT files/image.png "PUT /files/image.png HTTP/1.1" 200 - 234 65957 365 319 "-" "" -'
+    @file_format.should parse_line(line).as(:access).and_capture(
+        :bucket_owner    => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a', 
+        :bucket          => 'testbucket', 
+        :timestamp       => 20100303230459, 
+        :remote_ip       => '174.119.31.76', 
+        :requester       => '09216466b5571a8db0bf5abca72041fd3fc163e5eb83c51159735353ac6a2b9a',
+        :key             => 'files/image.png', 
+        :operation       => 'REST.COPY.OBJECT', 
+        :total_time      => 0.365, 
+        :turnaround_time => 0.319, 
+        :bytes_sent      => 234, 
+        :object_size     => 65957, 
+        :referer         => nil,
+        :user_agent      => '')
+  end
+
 end

data/spec/unit/file_format/rails3_format_spec.rb

@@ -0,0 +1,120 @@
+require 'spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Rails do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:rails3).should be_valid
+  end
+  
+  describe '#parse_line' do
+    before(:each) { @file_format = RequestLogAnalyzer::FileFormat.load(:rails3) }
+
+    it "should parse :started lines correctly" do
+      line = 'Started GET "/queries" for 127.0.0.1 at 2010-02-25 16:15:18'
+      @file_format.should parse_line(line).as(:started).and_capture(:method => 'GET', 
+            :path => '/queries', :ip => '127.0.0.1', :timestamp => 20100225161518)
+    end
+    
+    it "should parse :processing lines correctly" do
+      line = ' Processing by QueriesController#index as HTML'
+      @file_format.should parse_line(line).as(:processing).and_capture(
+        :controller => 'QueriesController', :action => 'index', :format => 'HTML')
+    end
+    
+    
+    # it "should parse beta :completed lines correctly" do
+    #   line = 'Completed in 9ms (Views: 4.9ms | ActiveRecord: 0.5ms) with 200'
+    #   @file_format.should parse_line(line).as(:completed).and_capture(
+    #       :duration => 0.009, :status => 200)
+    # end
+    
+    it "should parse :completed lines correctly" do
+      line = 'Completed 200 OK in 170ms (Views: 78.4ms | ActiveRecord: 48.2ms)'
+      @file_format.should parse_line(line).as(:completed).and_capture(
+          :duration => 0.170, :status => 200)
+    end
+    
+    it "should pase :failure lines correctly" do
+      line = "ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:"
+      @file_format.should parse_line(line).as(:failure).and_capture(:line => 3, 
+          :error   => 'ActionView::Template::Error', 
+          :message => "undefined local variable or method `field' for #<Class>", 
+          :file    => '/Users/willem/Code/warehouse/app/views/queries/execute.csv.erb')
+    end
+  end
+  
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails3))
+    end
+    
+    it "should parse a successful request correctly" do
+      log = <<-EOLOG
+        Started GET "/" for 127.0.0.1 at 2010-03-19 06:40:41
+          Processing by QueriesController#index as HTML
+          SQL (16.3ms)  SHOW TABLES
+          Query Load (32.0ms)  SELECT `queries`.* FROM `queries`
+        Rendered queries/index.html.erb within layouts/default (40.9ms)
+        Completed 200 OK in 170ms (Views: 78.4ms | ActiveRecord: 48.2ms)
+      EOLOG
+      
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_io(StringIO.new(log)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails3::Request) && request.completed?
+      end
+    end
+    
+    it "should parse an unroutable request correctly" do
+      log = <<-EOLOG
+        Started GET "/404" for 127.0.0.1 at 2010-03-19 06:40:57
+
+        ActionController::RoutingError (No route matches "/404"):
+
+
+        Rendered /Users/rails/actionpack/lib/action_dispatch/middleware/templates/rescues/routing_error.erb within rescues/layout (1.0ms)
+      
+      EOLOG
+
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_io(StringIO.new(log)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails3::Request) && request.completed?
+      end
+    end
+    
+    it "should parse a failing request correctly" do
+      log = <<-EOLOG
+        Started POST "/queries/397638749/execute.csv" for 127.0.0.1 at 2010-03-01 18:44:33
+          Processing by QueriesController#execute as CSV
+          Parameters: {"commit"=>"Run query", "authenticity_token"=>"pz9WcxkcrlG/43eg6BgSAnJL7yIsaffuHbYxPHUsUzQ=", "id"=>"397638749"}
+
+        ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/application/app/views/queries/execute.csv.erb:
+        1: <%=raw @result.fields.map { |f| f.humanize.to_json }.join(',') %>
+        2: <% @result.each do |record| %>
+        3:   <%=raw @result.fields.map { |f| record[field].to_s }.join(",") %>
+        4: <% end %>
+
+            app/views/queries/execute.csv.erb:3:in `_render_template__652100315_2182241460_0'
+            app/views/queries/execute.csv.erb:3:in `map'
+            app/views/queries/execute.csv.erb:3:in `_render_template__652100315_2182241460_0'
+            app/views/queries/execute.csv.erb:2:in `_render_template__652100315_2182241460_0'
+            app/controllers/queries_controller.rb:34:in `execute'
+
+        Rendered /rails/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/_trace.erb (1.0ms)
+        Rendered /rails/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/_request_and_response.erb (9.7ms)
+        Rendered /rails/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/template_error.erb within /Users/willem/.rvm/gems/ruby-1.8.7-p248/gems/actionpack-3.0.0.beta/lib/action_dispatch/middleware/templates/rescues/layout.erb (20.4ms)
+
+      EOLOG
+
+      request_counter.should_receive(:hit!).once
+      @log_parser.should_not_receive(:warn)
+      
+      @log_parser.parse_io(StringIO.new(log)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails3::Request) && request.completed?
+      end
+    end
+  end
+end

data/tasks/github-gem.rake

@@ -119,19 +119,19 @@ module GithubGem
         checks = [:check_current_branch, :check_clean_status, :check_not_diverged, :check_version]
         checks.unshift('spec:basic') if has_specs?
         checks.unshift('test:basic') if has_tests?
-        checks.push << [:check_rubyforge] if gemspec.rubyforge_project
+        # checks.push << [:check_rubyforge] if gemspec.rubyforge_project
 
         desc "Perform all checks that would occur before a release"
         task(:release_checks => checks)
 
         release_tasks = [:release_checks, :set_version, :build, :github_release, :gemcutter_release]
-        release_tasks << [:rubyforge_release] if gemspec.rubyforge_project
+        # release_tasks << [:rubyforge_release] if gemspec.rubyforge_project
 
         desc "Release a new verison of the gem"
         task(:release => release_tasks) { release_task }
 
-        task(:check_rubyforge)   { check_rubyforge_task }
-        task(:rubyforge_release) { rubyforge_release_task }
+        # task(:check_rubyforge)   { check_rubyforge_task }
+        # task(:rubyforge_release) { rubyforge_release_task }
         task(:gemcutter_release) { gemcutter_release_task }
         task(:github_release => [:commit_modified_files, :tag_version]) { github_release_task }
         task(:tag_version) { tag_version_task }
@@ -216,19 +216,19 @@ module GithubGem
       git.push(remote, remote_branch, true)
     end
 
-    # Checks whether Rubyforge is configured properly
-    def check_rubyforge_task
-      # Login no longer necessary when using rubyforge 2.0.0 gem
-      # raise "Could not login on rubyforge!" unless `rubyforge login 2>&1`.strip.empty?
-      output = `rubyforge names`.split("\n")
-      raise "Rubyforge group not found!"   unless output.any? { |line| %r[^groups\s*\:.*\b#{Regexp.quote(gemspec.rubyforge_project)}\b.*] =~ line }
-      raise "Rubyforge package not found!" unless output.any? { |line| %r[^packages\s*\:.*\b#{Regexp.quote(gemspec.name)}\b.*] =~ line }
-    end
-
-    # Task to release the .gem file toRubyforge.
-    def rubyforge_release_task
-      sh 'rubyforge', 'add_release', gemspec.rubyforge_project, gemspec.name, gemspec.version.to_s, "pkg/#{gemspec.name}-#{gemspec.version}.gem"
-    end
+    # # Checks whether Rubyforge is configured properly
+    # def check_rubyforge_task
+    #   # Login no longer necessary when using rubyforge 2.0.0 gem
+    #   # raise "Could not login on rubyforge!" unless `rubyforge login 2>&1`.strip.empty?
+    #   output = `rubyforge names`.split("\n")
+    #   raise "Rubyforge group not found!"   unless output.any? { |line| %r[^groups\s*\:.*\b#{Regexp.quote(gemspec.rubyforge_project)}\b.*] =~ line }
+    #   raise "Rubyforge package not found!" unless output.any? { |line| %r[^packages\s*\:.*\b#{Regexp.quote(gemspec.name)}\b.*] =~ line }
+    # end
+
+    # # Task to release the .gem file toRubyforge.
+    # def rubyforge_release_task
+    #   sh 'rubyforge', 'add_release', gemspec.rubyforge_project, gemspec.name, gemspec.version.to_s, "pkg/#{gemspec.name}-#{gemspec.version}.gem"
+    # end
     
     def gemcutter_release_task
       sh "gem push pkg/#{gemspec.name}-#{gemspec.version}.gem"

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.6.2
+  prerelease: false
+  segments: 
+  - 1
+  - 6
+  - 3
+  version: 1.6.3
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,29 +15,37 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-29 00:00:00 +01:00
+date: 2010-03-19 00:00:00 -04:00
 default_executable: request-log-analyzer
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 2
+        - 4
         version: 1.2.4
-    version: 
+  type: :development
+  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: git
-  type: :development
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 1
+        - 0
         version: 1.1.0
-    version: 
+  type: :development
+  version_requirements: *id002
 description: "    Request log analyzer's purpose is to find out how your web application is being used, how it performs and to\n    focus your optimization efforts. This tool will parse all requests in the application's log file and aggregate the \n    information. Once it is finished parsing the log file(s), it will show the requests that take op most server time \n    using various metrics. It can also insert all parsed request information into a database so you can roll your own\n    analysis. It supports Rails-, Merb- and Rack-based applications logs, Apache and Amazon S3 access logs and MySQL \n    slow query logs out of the box, but file formats of other applications can easily be supported by supplying an \n    easy to write log file format definition.\n"
 email: 
 - willem@railsdoctors.com
@@ -65,8 +78,8 @@ files:
 - lib/request_log_analyzer/tracker/frequency.rb
 - lib/request_log_analyzer/tracker/duration.rb
 - tasks/github-gem.rake
-- .gitignore
 - spec/unit/filter/filter_spec.rb
+- .gitignore
 - spec/lib/macros.rb
 - spec/spec_helper.rb
 - spec/fixtures/decompression.log.bz2
@@ -92,7 +105,9 @@ files:
 - spec/lib/mocks.rb
 - spec/unit/file_format/apache_format_spec.rb
 - spec/fixtures/syslog_1x.log
+- spec/unit/file_format/rails3_format_spec.rb
 - spec/integration/munin_plugins_rails_spec.rb
+- lib/request_log_analyzer/file_format/rails3.rb
 - spec/lib/helpers.rb
 - spec/unit/file_format/delayed_job_format_spec.rb
 - spec/fixtures/rails.db
@@ -144,8 +159,8 @@ files:
 - lib/request_log_analyzer/file_format/postgresql.rb
 - lib/request_log_analyzer/tracker/timespan.rb
 - spec/unit/controller/log_processor_spec.rb
-- spec/fixtures/sinatra.log
 - spec/unit/filter/field_filter_spec.rb
+- spec/fixtures/sinatra.log
 - spec/fixtures/merb_prefixed.log
 - lib/request_log_analyzer/log_processor.rb
 - lib/request_log_analyzer/filter/timespan.rb
@@ -189,18 +204,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: 
 - To use the database inserter, ActiveRecord and an appropriate database adapter are required.
 rubyforge_project: r-l-a
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers
@@ -219,6 +236,7 @@ test_files:
 - spec/integration/mailer_spec.rb
 - spec/unit/tracker/frequency_tracker_spec.rb
 - spec/unit/file_format/apache_format_spec.rb
+- spec/unit/file_format/rails3_format_spec.rb
 - spec/integration/munin_plugins_rails_spec.rb
 - spec/unit/file_format/delayed_job_format_spec.rb
 - spec/unit/source/log_parser_spec.rb