request-log-analyzer 1.7.0 → 1.8.0

data/bin/request-log-analyzer

@@ -34,6 +34,7 @@ begin
     command_line.option(:file, :alias => :e)
     command_line.option(:mail, :alias => :m)
     command_line.option(:mailhost, :default => 'localhost')
+    command_line.option(:mailsubject)    
     command_line.option(:parse_strategy, :default => 'assume-correct')
     command_line.option(:yaml)
     command_line.option(:dump) # To be deprecated
@@ -85,6 +86,7 @@ rescue CommandLine::Error => e
   puts "  --file <filename>          Redirect output to file."
   puts "  --mail <emailaddress>      Send report to an email address."
   puts "  --mailhost <server>        Use the given server as the SMTP server for sending email."
+  puts "  --mailsubject <text>       Overwrite default mailsubject."
   puts "  --no-progress              Hide the progress bar."
   puts "  --output <format>          Output format. Supports 'html' and 'fixed_width'."
   puts "  --report-width <amount>    Width of ASCII report. Defaults to terminal width."

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
 
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.7.0"
+  VERSION = "1.8.0"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/lib/request_log_analyzer/controller.rb

@@ -21,7 +21,7 @@ module RequestLogAnalyzer
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
     def self.build_from_arguments(arguments)
       
-      require 'lib/mixins/gets_memory_protection' if arguments[:gets_memory_protection]
+      require File.dirname(__FILE__) + '/../mixins/gets_memory_protection' if arguments[:gets_memory_protection]
       
       options = {}
       
@@ -45,6 +45,7 @@ module RequestLogAnalyzer
       options[:report_sort]    = arguments[:report_sort]
       options[:report_amount]  = arguments[:report_amount]
       options[:mailhost]       = arguments[:mailhost] 
+      options[:mailsubject]    = arguments[:mailsubject]
       options[:silent]         = arguments[:silent] 
       
       # Apache format workaround
@@ -107,6 +108,7 @@ module RequestLogAnalyzer
     # * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, :amazon_s3, :mysql or RequestLogAnalyzer::FileFormat class. (Defaults to :rails).
     # * <tt>:mail</tt> Email the results to this email address.
     # * <tt>:mailhost</tt> Email the results to this mail server.
+    # * <tt>:mailsubject</tt> Email subject.
     # * <tt>:no_progress</tt> Do not display the progress bar (increases parsing speed).
     # * <tt>:output</tt> 'FixedWidth', 'HTML' or RequestLogAnalyzer::Output class. Defaults to 'FixedWidth'.
     # * <tt>:reject</tt> Reject specific {:field => :value} combination (expects a single hash).
@@ -163,7 +165,7 @@ module RequestLogAnalyzer
         output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
         output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
       elsif options[:mail]
-        output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost])
+        output_object = RequestLogAnalyzer::Mailer.new(options[:mail], options[:mailhost], :subject => options[:mailsubject])
         output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount  }
       else
         output_object = STDOUT

data/lib/request_log_analyzer/file_format.rb

@@ -20,9 +20,10 @@ module RequestLogAnalyzer::FileFormat
       # a usable class is provided. Use this format class.
       klass = file_format
 
-    elsif file_format.kind_of?(String) && File.exist?(file_format)
+    elsif file_format.kind_of?(String) && File.exist?(file_format) && File.file?(file_format)
       # load a format from a ruby file
       require file_format
+
       const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
       if RequestLogAnalyzer::FileFormat.const_defined?(const)
         klass = RequestLogAnalyzer::FileFormat.const_get(const)

data/lib/request_log_analyzer/file_format/delayed_job2.rb

@@ -0,0 +1,38 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The DelayedJob2 file format parsed log files that are created by DelayedJob 2.0 or higher.
+  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
+  class DelayedJob2 < Base
+    
+    extend CommonRegularExpressions
+    
+    line_definition :job_lock do |line|
+      line.header = true
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\w+ host:(\S+) pid:(\d+)\)\] acquired lock on (\S+)/
+      
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:host)
+      line.capture(:pid).as(:integer)
+      line.capture(:job)
+    end
+    
+    line_definition :job_completed do |line|
+      line.footer = true
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \w+ host:(\S+) pid:(\d+) completed after (\d+\.\d+)/
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:host)
+      line.capture(:pid).as(:integer)
+      line.capture(:duration).as(:duration, :unit => :sec)
+    end
+    
+    report do |analyze|
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
+      #analyze.frequency :job, :if => lambda { |request| request[:attempts] ==  1 }, :title => "Failed jobs"
+      
+      analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/oink.rb

@@ -0,0 +1,78 @@
+class RequestLogAnalyzer::FileFormat::Oink < RequestLogAnalyzer::FileFormat::Rails
+  # capture the PID
+  HODEL3000_PROCESSING = /\[(\d+)\]: Processing ((?:\w+::)*\w+)#(\w+)(?: to (\w+))? \(for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})\) \[([A-Z]+)\]/
+  
+  LINE_DEFINITIONS[:processing] = RequestLogAnalyzer::LineDefinition.new(:processing, :header => true,
+          :teaser   => /Processing /,
+          :regexp   => Regexp.union(LINE_DEFINITIONS[:processing].regexp,HODEL3000_PROCESSING),
+          :captures => [{ :name => :controller, :type  => :string }, # Default Rails Processing
+                        { :name => :action,     :type  => :string },
+                        { :name => :format,     :type  => :string, :default => 'html' },
+                        { :name => :ip,         :type  => :string },
+                        { :name => :timestamp,  :type  => :timestamp },
+                        { :name => :method,     :type  => :string },
+                        { :name => :pid, :type => :integer },        # Hodel 3000 Processing w/PID
+                        { :name => :controller, :type  => :string },
+                        { :name => :action,     :type  => :string },
+                        { :name => :format,     :type  => :string, :default => 'html' },
+                        { :name => :ip,         :type  => :string },
+                        { :name => :timestamp,  :type  => :timestamp },
+                        { :name => :method,     :type  => :string }])
+  
+  # capture the memory usage
+  LINE_DEFINITIONS[:memory_usage] = RequestLogAnalyzer::LineDefinition.new(:memory_usage,
+   :regexp   => /\[(\d+)\]: Memory usage: (\d+) /,
+   :captures => [{ :name => :pid, :type => :integer },{ :name => :memory, :type => :traffic }])
+  
+  # capture :memory usage in all line collections
+  LINE_COLLECTIONS.each { |k,v| LINE_COLLECTIONS[k] << :memory_usage }
+  
+  report(:append) do |analyze|
+      analyze.traffic :memory_diff, :category => REQUEST_CATEGORIZER, :title => "Largest Memory Increases", :line_type => :memory_usage
+  end
+  
+  # Keep a record of PIDs and their memory usage when validating requests.
+  def pids
+    @pids ||= {}
+  end
+  
+  class Request
+   # Overrides the #validate method to handle PID updating.
+   def validate
+     update_pids
+     super
+   end
+   
+   # Accessor for memory information associated with the specified request PID. If no memory exists
+   # for this request's :pid, the memory tracking is initialized.
+   def pid_memory
+     file_format.pids[self[:pid]] ||= { :last_memory_reading => -1, :current_memory_reading => -1 }
+   end
+    
+   # Calculates :memory_diff for each request based on the last completed request that was not a failure.
+   def update_pids
+     # memory isn't recorded with exceptions. need to set #last_memory_reading+ to -1 as
+     # the memory used could have changed. for the next request the memory change will not be recorded.
+     #
+     # NOTE - the failure regex was not matching with a Rails Development log file.
+     if has_line_type?(:failure) and processing = has_line_type?(:processing)
+       pid_memory[:last_memory_reading] = -1
+     elsif mem_line = has_line_type?(:memory_usage)
+        memory_reading = mem_line[:memory]
+        pid_memory[:current_memory_reading] = memory_reading
+        # calcuate the change in memory
+        unless pid_memory[:current_memory_reading] == -1 || pid_memory[:last_memory_reading] == -1
+          # logged as kB, need to convert to bytes for the :traffic Tracker
+          memory_diff = (pid_memory[:current_memory_reading] - pid_memory[:last_memory_reading])*1024
+          if memory_diff > 0
+            self.attributes[:memory_diff] = memory_diff
+          end # if memory_diff > 0
+        end # unless
+        
+        pid_memory[:last_memory_reading] = pid_memory[:current_memory_reading]
+        pid_memory[:current_memory_reading] = -1
+      end # if mem_line
+      return true
+   end # def update_pids
+  end # class Request
+end

data/lib/request_log_analyzer/request.rb

@@ -30,11 +30,18 @@ module RequestLogAnalyzer
 
       # Converts :eval field, which should evaluate to a hash.
       def convert_eval(value, capture_definition)
-        eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
-      rescue SyntaxError
+        eval(sanitize_parameters(value)).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
+			# Wide range of errors possible with wild eval. ATM we choose to crash on this.
+      rescue SyntaxError 
         nil
       end
 
+      # Removes certain string sequences which would be problematic for eval.
+      # TODO remove all characters not valid in ruby symbols
+      def sanitize_parameters(parameter_string)
+        parameter_string.gsub(/#</, '"').gsub(/>,/, '", ').gsub(/\\0/, '')
+      end
+
       # Slow default method to parse timestamps.
       # Reimplement this function in a file format specific Request class
       # to improve the timestamp parsing speed.
@@ -177,4 +184,4 @@ module RequestLogAnalyzer
       @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.max
     end
   end
-end
+end

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.7.0"
-  s.date    = "2010-04-28"
+  s.version = "1.8.0"
+  s.date    = "2010-06-30"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -36,6 +36,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(spec/unit/file_format/delayed_job_format_spec.rb spec/fixtures/rails_unordered.log spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/output/fancy_html.rb README.rdoc lib/cli/progressbar.rb spec/fixtures/rails.db lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/fixtures/header_and_footer.log lib/mixins/gets_memory_protection.rb spec/fixtures/decompression.log spec/fixtures/rails_1x.log lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/tracker/numeric_value.rb spec/unit/file_format/line_definition_spec.rb spec/unit/request_spec.rb spec/unit/filter/filter_spec.rb spec/unit/aggregator/summarizer_spec.rb lib/request_log_analyzer/file_format/rails.rb spec/database.yml spec/unit/tracker/numeric_value_tracker_spec.rb spec/spec_helper.rb lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb spec/fixtures/sinatra.log lib/request_log_analyzer/database/warning.rb spec/fixtures/merb_prefixed.log spec/unit/file_format/postgresql_format_spec.rb lib/request_log_analyzer/filter/timespan.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/format_autodetection_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb LICENSE lib/request_log_analyzer/file_format.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb spec/fixtures/merb.log lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/mailer_spec.rb spec/fixtures/postgresql.log spec/lib/mocks.rb lib/request_log_analyzer/tracker/hourly_spread.rb spec/fixtures/syslog_1x.log spec/unit/file_format/rails3_format_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/file_format/rails3.rb spec/unit/controller/log_processor_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/filter/field_filter_spec.rb DESIGN.rdoc lib/request_log_analyzer/log_processor.rb spec/fixtures/multiple_files_1.log spec/unit/source/log_parser_spec.rb spec/fixtures/decompression.tar.gz spec/unit/database/base_class_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb tasks/request_log_analyzer.rake lib/request_log_analyzer/tracker/duration.rb spec/fixtures/test_language_combined.log bin/request-log-analyzer lib/request_log_analyzer/tracker.rb tasks/github-gem.rake lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb .gitignore lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/database/request.rb lib/cli/database_console_init.rb lib/request_log_analyzer/database/source.rb spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/filter.rb spec/fixtures/apache_combined.log spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log Rakefile spec/unit/tracker/traffic_tracker_spec.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/tracker/timespan.rb spec/lib/macros.rb spec/integration/scout_spec.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/unit/tracker/timespan_tracker_spec.rb spec/lib/testing_format.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/request.rb spec/fixtures/multiple_files_2.log spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz lib/request_log_analyzer/file_format/rack.rb spec/integration/munin_plugins_rails_spec.rb spec/fixtures/mysql_slow_query.log spec/lib/helpers.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/file_format/apache_format_spec.rb)
-  s.test_files = %w(spec/unit/file_format/delayed_job_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/request_spec.rb spec/unit/filter/filter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/database/connection_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/mailer_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/integration/scout_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/file_format/apache_format_spec.rb)
+  s.files      = %w(spec/unit/file_format/delayed_job_format_spec.rb spec/fixtures/rails_unordered.log spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/output/fancy_html.rb README.rdoc lib/cli/progressbar.rb spec/fixtures/rails.db lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/unit/file_format/delayed_job2_format_spec.rb spec/fixtures/test_order.log spec/fixtures/header_and_footer.log lib/mixins/gets_memory_protection.rb spec/fixtures/decompression.log spec/fixtures/rails_1x.log lib/request_log_analyzer/file_format/apache.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/tracker/numeric_value.rb spec/unit/file_format/line_definition_spec.rb spec/unit/request_spec.rb spec/unit/filter/filter_spec.rb spec/unit/aggregator/summarizer_spec.rb lib/request_log_analyzer/file_format/rails.rb spec/database.yml spec/unit/tracker/numeric_value_tracker_spec.rb spec/spec_helper.rb lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb spec/fixtures/oink_22.log spec/fixtures/sinatra.log lib/request_log_analyzer/database/warning.rb spec/fixtures/merb_prefixed.log spec/unit/file_format/postgresql_format_spec.rb lib/request_log_analyzer/filter/timespan.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/format_autodetection_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/file_format/oink.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb LICENSE lib/request_log_analyzer/file_format.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb spec/fixtures/merb.log lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/mailer_spec.rb spec/fixtures/postgresql.log spec/lib/mocks.rb lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/file_format/rails3_format_spec.rb spec/fixtures/syslog_1x.log lib/request_log_analyzer/source.rb lib/request_log_analyzer/file_format/rails3.rb spec/unit/controller/log_processor_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/filter/field_filter_spec.rb DESIGN.rdoc lib/request_log_analyzer/log_processor.rb spec/fixtures/multiple_files_1.log spec/unit/source/log_parser_spec.rb spec/fixtures/decompression.tar.gz spec/unit/database/base_class_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/source/database_loader.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb tasks/request_log_analyzer.rake lib/request_log_analyzer/tracker/duration.rb spec/fixtures/test_language_combined.log bin/request-log-analyzer lib/request_log_analyzer/tracker.rb tasks/github-gem.rake lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb .gitignore lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/database/request.rb lib/cli/database_console_init.rb lib/request_log_analyzer/database/source.rb spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/filter.rb spec/fixtures/apache_combined.log spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/aggregator/summarizer.rb spec/fixtures/oink_22_failure.log spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/file_format/delayed_job2.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/mysql_format_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log Rakefile spec/unit/tracker/traffic_tracker_spec.rb lib/request_log_analyzer/file_format/postgresql.rb lib/request_log_analyzer/tracker/timespan.rb spec/lib/macros.rb spec/integration/scout_spec.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/unit/tracker/timespan_tracker_spec.rb spec/lib/testing_format.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/request.rb spec/unit/file_format/oink_format_spec.rb spec/fixtures/multiple_files_2.log spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz lib/request_log_analyzer/file_format/rack.rb spec/integration/munin_plugins_rails_spec.rb spec/fixtures/mysql_slow_query.log spec/lib/helpers.rb lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/file_format/apache_format_spec.rb)
+  s.test_files = %w(spec/unit/file_format/delayed_job_format_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/delayed_job2_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/request_spec.rb spec/unit/filter/filter_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/database/connection_spec.rb spec/unit/file_format/postgresql_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/mailer_spec.rb spec/unit/file_format/rails3_format_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/integration/scout_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/file_format/oink_format_spec.rb spec/unit/database/database_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/file_format/apache_format_spec.rb)
 end

data/spec/fixtures/oink_22.log

@@ -0,0 +1,19 @@
+Jun 18 11:27:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:27:37) [GET]
+Jun 18 11:27:37 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016
+Jun 18 11:27:37 derek rails[67783]: Parameters: {"id"=>"5171"}
+Jun 18 11:27:37 derek rails[67783]: Memory usage: 400000 | PID: 67783
+Jun 18 11:27:37 derek rails[67783]: Completed in 207ms (View: 0, DB: 258) | 200 OK [http://localhost/file_records/inline/5171]
+
+Jun 18 11:28:11 derek rails[700]: Processing InfoController#about (for 127.0.0.1 at 2010-06-18 11:28:11) [GET]
+Jun 18 11:28:12 derek rails[700]: Memory usage: 700000 | PID: 67783
+Jun 18 11:28:12 derek rails[700]: Completed in 379ms (View: 89, DB: 258) | 200 OK [http://localhost/info/about]
+
+Jun 18 11:28:11 derek rails[700]: Processing InfoController#board (for 127.0.0.1 at 2010-06-18 11:28:11) [GET]
+Jun 18 11:28:12 derek rails[700]: Memory usage: 750000 | PID: 67783
+Jun 18 11:28:12 derek rails[700]: Completed in 379ms (View: 89, DB: 258) | 200 OK [http://localhost/info/board]
+
+Jun 18 11:31:19 derek rails[67783]: Processing InnovationsController#innovation_param (for 127.0.0.1 at 2010-06-18 11:31:19) [GET]
+Jun 18 11:31:19 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016
+Jun 18 11:31:19 derek rails[67783]: Parameters: {"organization_param"=>"harvard", "innovation_param"=>"potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti"}
+Jun 18 11:31:31 derek rails[67783]: Memory usage: 430000 | PID: 67783
+Jun 18 11:31:31 derek rails[67783]: Completed in 11922ms (View: 200, DB: 11673) | 200 OK [http://localhost/harvard/potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti]

data/spec/fixtures/oink_22_failure.log

@@ -0,0 +1,17 @@
+Jun 18 11:25:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:25:37) [GET]
+Jun 18 11:25:40 derek rails[67783]: NoMethodError (undefined method `update_domain_account' for nil:NilClass):
+
+Jun 18 11:27:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:27:37) [GET]
+Jun 18 11:27:37 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016
+Jun 18 11:27:37 derek rails[67783]: Parameters: {"id"=>"5171"}
+Jun 18 11:27:37 derek rails[67783]: Memory usage: 400000 | PID: 67783
+Jun 18 11:27:37 derek rails[67783]: Completed in 207ms (View: 0, DB: 258) | 200 OK [http://localhost/file_records/inline/5171]
+
+Jun 18 11:30:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:27:37) [GET]
+Jun 18 11:30:40 derek rails[67783]: NoMethodError (undefined method `update_domain_account' for nil:NilClass):
+
+Jun 18 11:31:19 derek rails[67783]: Processing InnovationsController#innovation_param (for 127.0.0.1 at 2010-06-18 11:31:19) [GET]
+Jun 18 11:31:19 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016
+Jun 18 11:31:19 derek rails[67783]: Parameters: {"organization_param"=>"harvard", "innovation_param"=>"potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti"}
+Jun 18 11:31:31 derek rails[67783]: Memory usage: 430000 | PID: 67783
+Jun 18 11:31:31 derek rails[67783]: Completed in 11922ms (View: 200, DB: 11673) | 200 OK [http://localhost/harvard/potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti]

data/spec/integration/command_line_usage_spec.rb

@@ -83,7 +83,7 @@ describe RequestLogAnalyzer, 'running from command line' do
   end
 
   it "should dump the results to a YAML file" do
-    run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:yaml)}")
+    run("#{log_fixture(:rails_1x)} --yaml #{temp_output_file(:yaml)}")
     File.exist?(temp_output_file(:yaml)).should be_true
     YAML::load(File.read(temp_output_file(:yaml))).should have_at_least(1).item
   end

data/spec/integration/mailer_spec.rb

@@ -30,9 +30,25 @@ describe RequestLogAnalyzer, 'running mailer integration' do
     find_string_in_file("From: <contact@railsdoctors.com>", @log_file).should_not be_nil
     find_string_in_file("To: <root@localhost>", @log_file).should_not be_nil
     find_string_in_file("From: Request-log-analyzer reporter <contact@railsdoctors.com>", @log_file).should_not be_nil
+    find_string_in_file("Subject: Request log analyzer report - generated on", @log_file).should_not be_nil
     find_string_in_file("Request summary", @log_file).should_not be_nil
     find_string_in_file("PeopleController#show.html [ |    1 |  0.29s |  0.29s |  0.00s |  0.29s |  0.29s", @log_file).should_not be_nil
   end
+  
+  it "should allow a custom mail subject" do
+    RequestLogAnalyzer::Controller.build(
+      :mail         => 'root@localhost',
+      :mailhost     => 'localhost:2525',
+      :mailsubject  => 'TESTSUBJECT',
+      :source_files => log_fixture(:rails_1x),
+      :format       => RequestLogAnalyzer::FileFormat::Rails,
+      :no_progress  => true
+    ).run!
+    
+    Process.wait # Wait for mailer to complete
+
+    find_string_in_file("Subject: TESTSUBJECT", @log_file).should_not be_nil  
+  end
 
   it "should send html mail" do    
     RequestLogAnalyzer::Controller.build(

data/spec/unit/file_format/delayed_job2_format_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::DelayedJob do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:delayed_job).should be_valid
+  end
+
+  describe '#parse_line' do
+    
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:delayed_job2)
+    end
+
+    it "should parse a :job_lock line correctly" do
+      line = "2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob"
+      @file_format.should parse_line(line).as(:job_lock).and_capture(:timestamp => 20100517173734,
+                                    :job => 'S3FileJob', :host => 'hostname.co.uk', :pid => 11888)
+    end
+
+    it "should parse a :job_completed line correctly" do
+      line = '2010-05-17T17:37:35+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.0676'
+      @file_format.should parse_line(line).as(:job_completed).and_capture(:timestamp => 20100517173735,
+                                    :duration => 1.0676, :host => 'hostname.co.uk', :pid => 11888)
+    end
+  end
+  
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:delayed_job2))
+    end
+    
+    it "should parse a batch of completed jobs without warnings" do
+      fragment = <<-EOLOG
+        2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888
+        2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
+        2010-05-17T17:37:35+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.0676
+        2010-05-17T17:37:35+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
+        2010-05-17T17:37:37+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.4407
+        2010-05-17T17:37:37+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob
+        2010-05-17T17:37:44+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 6.9374
+        2010-05-17T17:37:44+0000: 3 jobs processed at 0.3163 j/s, 0 failed ...
+        2010-05-19T11:47:26+0000: Exiting...
+      EOLOG
+
+      request_counter.should_receive(:hit!).exactly(3).times
+      @log_parser.should_not_receive(:warn)
+
+      @log_parser.parse_io(StringIO.new(fragment)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::DelayedJob2::Request)
+      end
+    end
+  end
+end

data/spec/unit/file_format/delayed_job_format_spec.rb

@@ -28,7 +28,7 @@ describe RequestLogAnalyzer::FileFormat::DelayedJob do
         :failed_job => 'BackgroundJob::ThumbnailSaver', :exception => 'ActiveRecord::RecordNotFound')
     end
     
-    it "" do
+    it "should parse a failed job lock line correctly" do
       line = "* [JOB] failed to acquire exclusive lock for BackgroundJob::ThumbnailSaver"
       @file_format.should parse_line(line).as(:job_lock_failed).and_capture(:locked_job => 'BackgroundJob::ThumbnailSaver')
     end

data/spec/unit/file_format/oink_format_spec.rb

@@ -0,0 +1,98 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::Oink do
+  describe '.create' do
+
+    context 'without providing a lines argument' do
+      before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink) }
+
+      it "should create a valid file format" do
+        @oink.should be_valid
+      end
+      
+      it "should parse :memory_usage line" do
+        @oink.line_definitions.should include(:memory_usage)
+      end
+
+    end
+  
+  end
+  
+  describe '#parse_line' do
+    before(:each) { @oink = RequestLogAnalyzer::FileFormat.load(:oink, :all) }
+    
+    it "should parse a :memory_usage line correctly" do
+      line = 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783'
+      @oink.should parse_line(line).as(:memory_usage).and_capture(:pid => 67783, :memory => 714052)
+    end
+    
+    it "should parse the PID from a :processing line correctly" do
+      line = 'Aug 14 21:16:30 derek rails[67783]: Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]'
+      @oink.should parse_line(line).as(:processing).and_capture(:pid => 67783, :controller => 'PeopleController', :action => 'index', :timestamp => 20080814211630, :method => 'GET', :ip => '1.1.1.1')
+    end
+  end
+  
+  describe '#parse_io' do
+    context "Rails 2.2 style log" do
+      before(:each) do
+        @log_parser = RequestLogAnalyzer::Source::LogParser.new(
+              RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
+      end
+
+      it "should parse requests" do
+        request_counter.should_receive(:hit!).exactly(4).times
+        
+        @log_parser.parse_file(log_fixture(:oink_22)) do |request|
+          request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
+        end
+      end
+      
+      it "should not record :memory_diff on first request" do
+        @log_parser.parse_file(log_fixture(:oink_22)) do |request|
+          if @log_parser.parsed_requests == 1
+            request[:memory_diff].should == nil
+          end
+        end
+      end
+      
+      it "should record :memory_diff of 2nd tracked PID" do
+        @log_parser.parse_file(log_fixture(:oink_22)) do |request|
+          if @log_parser.parsed_requests == 3
+            request[:memory_diff].should == 50000*1024
+          end
+        end
+      end
+      
+      it "should record :memory_diff of 1st tracked PID" do
+        @log_parser.parse_file(log_fixture(:oink_22)) do |request|
+          if @log_parser.parsed_requests == 4
+            request[:memory_diff].should == 30000*1024
+          end
+        end
+      end
+    end
+    
+    context 'Rails 2.2 style log w/failure' do
+      before(:each) do
+        @log_parser = RequestLogAnalyzer::Source::LogParser.new(
+              RequestLogAnalyzer::FileFormat.load(:oink), :parse_strategy => 'cautious')
+      end
+      
+      it "should parse requests" do
+        request_counter.should_receive(:hit!).exactly(4).times
+        
+        @log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
+          request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::Rails::Request) && request.completed?
+        end
+      end
+      
+      it "should ignore memory changes when a failure occurs" do
+        @log_parser.parse_file(log_fixture(:oink_22_failure)) do |request|
+          if @log_parser.parsed_requests == 4
+            request[:memory_diff].should == nil
+          end
+        end
+      end
+    end
+  end
+end

data/tasks/github-gem.rake

@@ -127,8 +127,22 @@ module GithubGem
         release_tasks = [:release_checks, :set_version, :build, :github_release, :gemcutter_release]
         # release_tasks << [:rubyforge_release] if gemspec.rubyforge_project
 
-        desc "Release a new verison of the gem"
+        desc "Release a new version of the gem using the VERSION environment variable"
         task(:release => release_tasks) { release_task }
+        
+        namespace(:release) do
+          desc "Release the next version of the gem, by incrementing the last version segment by 1"
+          task(:next => [:next_version] + release_tasks) { release_task }
+
+          desc "Release the next version of the gem, using a bump increment (0.0.1)"
+          task(:bump => [:next_bump_version] + release_tasks) { release_task }
+
+          desc "Release the next version of the gem, using a minor increment (0.1.0)"
+          task(:minor => [:next_minor_version] + release_tasks) { release_task }
+
+          desc "Release the next version of the gem, using a major increment (1.0.0)"
+          task(:major => [:next_major_version] + release_tasks) { release_task }
+        end
 
         # task(:check_rubyforge)   { check_rubyforge_task }
         # task(:rubyforge_release) { rubyforge_release_task }
@@ -137,6 +151,11 @@ module GithubGem
         task(:tag_version) { tag_version_task }
         task(:commit_modified_files) { commit_modified_files_task }
 
+        task(:next_version)       { next_version_task }
+        task(:next_bump_version)  { next_version_task(:bump) }
+        task(:next_minor_version) { next_version_task(:minor) }
+        task(:next_major_version) { next_version_task(:major) }
+        
         desc "Updates the gem release tasks with the latest version on Github"
         task(:update_tasks) { update_tasks_task }
       end
@@ -160,6 +179,32 @@ module GithubGem
       sh "mv #{gemspec.name}-#{gemspec.version}.gem pkg/#{gemspec.name}-#{gemspec.version}.gem"
     end
 
+    def newest_version
+      git.tags.map { |tag| tag.name.split('-').last }.compact.map { |v| Gem::Version.new(v) }.max || Gem::Version.new('0.0.0')
+    end
+
+    def next_version(increment = nil)
+      next_version = newest_version.segments
+      increment_index = case increment
+        when :micro then 3
+        when :bump  then 2
+        when :minor then 1
+        when :major then 0
+        else next_version.length - 1
+      end
+      
+      next_version[increment_index] ||= 0
+      next_version[increment_index] = next_version[increment_index].succ
+      ((increment_index + 1)...next_version.length).each { |i| next_version[i] = 0 }
+      
+      Gem::Version.new(next_version.join('.'))
+    end
+
+    def next_version_task(increment = nil)
+      ENV['VERSION'] = next_version(increment).version
+      puts "Releasing version #{ENV['VERSION']}..."
+    end
+
     # Updates the version number in the gemspec file, the VERSION constant in the main
     # include file and the contents of the VERSION file.
     def version_task
@@ -173,9 +218,7 @@ module GithubGem
     def check_version_task
       raise "#{ENV['VERSION']} is not a valid version number!" if ENV['VERSION'] && !Gem::Version.correct?(ENV['VERSION'])
       proposed_version = Gem::Version.new(ENV['VERSION'] || gemspec.version)
-      # Loads the latest version number using the created tags
-      newest_version   = git.tags.map { |tag| tag.name.split('-').last }.compact.map { |v| Gem::Version.new(v) }.max
-      raise "This version (#{proposed_version}) is not higher than the highest tagged version (#{newest_version})" if newest_version && newest_version >= proposed_version
+      raise "This version (#{proposed_version}) is not higher than the highest tagged version (#{newest_version})" if newest_version >= proposed_version
     end
 
     # Checks whether the current branch is not diverged from the remote branch

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 1
-  - 7
+  - 8
   - 0
-  version: 1.7.0
+  version: 1.8.0
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-28 00:00:00 +02:00
+date: 2010-06-30 00:00:00 +02:00
 default_executable: request-log-analyzer
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -66,6 +66,7 @@ files:
 - spec/fixtures/rails.db
 - lib/request_log_analyzer/file_format/rails_development.rb
 - spec/fixtures/rails_22.log
+- spec/unit/file_format/delayed_job2_format_spec.rb
 - spec/fixtures/test_order.log
 - spec/fixtures/header_and_footer.log
 - lib/mixins/gets_memory_protection.rb
@@ -86,6 +87,7 @@ files:
 - spec/unit/aggregator/database_inserter_spec.rb
 - spec/fixtures/decompression.log.bz2
 - spec/unit/database/connection_spec.rb
+- spec/fixtures/oink_22.log
 - spec/fixtures/sinatra.log
 - lib/request_log_analyzer/database/warning.rb
 - spec/fixtures/merb_prefixed.log
@@ -97,6 +99,7 @@ files:
 - lib/request_log_analyzer/filter/field.rb
 - lib/request_log_analyzer/file_format/delayed_job.rb
 - lib/request_log_analyzer/file_format/merb.rb
+- lib/request_log_analyzer/file_format/oink.rb
 - lib/request_log_analyzer/filter/anonymize.rb
 - lib/request_log_analyzer/file_format/amazon_s3.rb
 - LICENSE
@@ -110,8 +113,8 @@ files:
 - spec/fixtures/postgresql.log
 - spec/lib/mocks.rb
 - lib/request_log_analyzer/tracker/hourly_spread.rb
-- spec/fixtures/syslog_1x.log
 - spec/unit/file_format/rails3_format_spec.rb
+- spec/fixtures/syslog_1x.log
 - lib/request_log_analyzer/source.rb
 - lib/request_log_analyzer/file_format/rails3.rb
 - spec/unit/controller/log_processor_spec.rb
@@ -150,13 +153,15 @@ files:
 - spec/unit/tracker/duration_tracker_spec.rb
 - spec/fixtures/test_file_format.log
 - lib/request_log_analyzer/aggregator/summarizer.rb
+- spec/fixtures/oink_22_failure.log
 - spec/unit/filter/timespan_filter_spec.rb
 - spec/unit/file_format/common_regular_expressions_spec.rb
 - spec/unit/file_format/merb_format_spec.rb
 - spec/unit/file_format/file_format_api_spec.rb
-- lib/request_log_analyzer/file_format/mysql.rb
+- lib/request_log_analyzer/file_format/delayed_job2.rb
 - spec/unit/file_format/rack_format_spec.rb
 - spec/unit/file_format/mysql_format_spec.rb
+- lib/request_log_analyzer/file_format/mysql.rb
 - spec/fixtures/decompression.log.gz
 - lib/request_log_analyzer/tracker/frequency.rb
 - request-log-analyzer.gemspec
@@ -176,6 +181,7 @@ files:
 - lib/request_log_analyzer/controller.rb
 - spec/unit/tracker/tracker_api_spec.rb
 - lib/request_log_analyzer/request.rb
+- spec/unit/file_format/oink_format_spec.rb
 - spec/fixtures/multiple_files_2.log
 - spec/unit/database/database_spec.rb
 - spec/fixtures/decompression.log.zip
@@ -225,6 +231,7 @@ summary: A command line tool to analyze request logs for Apache, Rails, Merb, My
 test_files: 
 - spec/unit/file_format/delayed_job_format_spec.rb
 - spec/unit/filter/anonymize_filter_spec.rb
+- spec/unit/file_format/delayed_job2_format_spec.rb
 - spec/unit/file_format/line_definition_spec.rb
 - spec/unit/request_spec.rb
 - spec/unit/filter/filter_spec.rb
@@ -258,6 +265,7 @@ test_files:
 - spec/unit/controller/controller_spec.rb
 - spec/unit/tracker/timespan_tracker_spec.rb
 - spec/unit/tracker/tracker_api_spec.rb
+- spec/unit/file_format/oink_format_spec.rb
 - spec/unit/database/database_spec.rb
 - spec/integration/munin_plugins_rails_spec.rb
 - spec/unit/file_format/apache_format_spec.rb