request-log-analyzer 1.5.3 → 1.5.4

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
 
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.5.3"
+  VERSION = "1.5.4"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/lib/request_log_analyzer/database.rb

@@ -1,5 +1,5 @@
 require 'rubygems'
-require 'activerecord'
+require 'active_record'
 
 class RequestLogAnalyzer::Database
 

data/lib/request_log_analyzer/file_format/delayed_job.rb

@@ -0,0 +1,51 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The DelayedJob file format parsed log files that are created by DelayedJob.
+  # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log
+  class DelayedJob < Base
+    
+    line_definition :job_lock do |line|
+      line.header = true
+      line.regexp = /\* \[JOB\] acquiring lock on (\S+)/
+      line.captures << { :name => :job, :type => :string }
+    end
+    
+    line_definition :job_completed do |line|
+      line.footer = true
+      line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/
+      line.captures << { :name => :completed_job, :type => :string } << 
+                       { :name => :duration, :type => :duration, :unit => :sec }
+    end
+    
+    line_definition :job_failed do |line|
+      line.footer = true
+      line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/
+      line.captures << { :name => :failed_job, :type => :string  } << 
+                       { :name => :exception,  :type => :string  } << 
+                       { :name => :attempts,   :type => :integer }
+      
+    end
+    
+    line_definition :job_lock_failed do |line|
+      line.footer = true
+      line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/
+      line.captures << { :name => :locked_job, :type => :string }
+    end
+    
+    # line_definition :batch_completed do |line|
+    #   line.header = true
+    #   line.footer = true
+    #   line.regexp = /(\d+) jobs processed at (\d+\.\d+) j\/s, (\d+) failed .../
+    #   line.captures << { :name => :total_amount,  :type => :integer } << 
+    #                    { :name => :mean_duration, :type => :duration, :unit => :sec } <<
+    #                    { :name => :failed_amount, :type => :integer }
+    # end
+    
+    report do |analyze|
+      analyze.frequency :job, :line_type => :job_completed, :title => "Completed jobs"
+      analyze.frequency :job, :if => lambda { |request| request[:attempts] ==  1 }, :title => "Failed jobs"
+      
+      analyze.duration :duration, :category => :job, :line_type => :job_completed, :title => "Job duration"
+    end
+  end
+end

data/request-log-analyzer.gemspec

@@ -2,8 +2,8 @@ Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
   
   # Do not set the version and date field manually, this is done by the release script
-  s.version = "1.5.3"
-  s.date    = "2009-12-14"
+  s.version = "1.5.4"
+  s.date    = "2009-12-17"
 
   s.rubyforge_project = 'r-l-a'
 
@@ -36,6 +36,6 @@ Gem::Specification.new do |s|
 
   # The files and test_files directives are set automatically by the release script.
   # Do not change them by hand, but make sure to add the files to the git repository.
-  s.files      = %w(spec/unit/tracker/hourly_spread_spec.rb lib/request_log_analyzer/output/html.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/unit/file_format/rack_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/integration/munin_plugins_rails_spec.rb spec/lib/helpers.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/tracker/numeric_value.rb spec/fixtures/test_language_combined.log lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml spec/unit/request_spec.rb lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/output/fancy_html.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log lib/request_log_analyzer/file_format/rails.rb Rakefile spec/unit/mailer_spec.rb lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/fixtures/sinatra.log spec/unit/filter/field_filter_spec.rb spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
-  s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/filter/filter_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
+  s.files      = %w(spec/unit/tracker/hourly_spread_spec.rb lib/request_log_analyzer/output/html.rb DESIGN.rdoc spec/fixtures/rails_unordered.log spec/fixtures/multiple_files_1.log lib/request_log_analyzer/database/source.rb README.rdoc spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/rails_22.log spec/fixtures/test_order.log spec/unit/database/base_class_spec.rb lib/request_log_analyzer/database/connection.rb lib/request_log_analyzer/aggregator.rb spec/fixtures/decompression.log lib/request_log_analyzer/file_format/apache.rb spec/unit/file_format/rack_format_spec.rb spec/fixtures/decompression.log.gz lib/request_log_analyzer/tracker/frequency.rb lib/request_log_analyzer/tracker/duration.rb tasks/github-gem.rake .gitignore spec/unit/filter/filter_spec.rb spec/lib/macros.rb spec/spec_helper.rb spec/fixtures/decompression.log.bz2 spec/unit/database/connection_spec.rb lib/request_log_analyzer/filter.rb spec/integration/scout_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/tracker/traffic.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/file_format/merb.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/file_format/amazon_s3.rb lib/cli/database_console.rb spec/integration/mailer_spec.rb lib/request_log_analyzer/line_definition.rb spec/unit/tracker/frequency_tracker_spec.rb spec/fixtures/multiple_files_2.log spec/lib/mocks.rb spec/unit/file_format/apache_format_spec.rb spec/fixtures/syslog_1x.log spec/integration/munin_plugins_rails_spec.rb spec/lib/helpers.rb spec/unit/file_format/delayed_job_format_spec.rb spec/fixtures/rails.db spec/unit/source/log_parser_spec.rb lib/request_log_analyzer/aggregator/database_inserter.rb spec/fixtures/header_and_footer.log spec/lib/testing_format.rb lib/request_log_analyzer/database.rb spec/unit/file_format/rails_format_spec.rb lib/request_log_analyzer/request.rb lib/request_log_analyzer/output/fixed_width.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/tracker/numeric_value.rb spec/fixtures/test_language_combined.log lib/cli/command_line_arguments.rb lib/request_log_analyzer.rb spec/database.yml spec/unit/request_spec.rb lib/request_log_analyzer/database/base.rb spec/unit/aggregator/database_inserter_spec.rb lib/request_log_analyzer/database/request.rb lib/request_log_analyzer/output/fancy_html.rb lib/cli/database_console_init.rb lib/cli/progressbar.rb lib/request_log_analyzer/database/warning.rb spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/apache_combined.log spec/unit/tracker/duration_tracker_spec.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/aggregator/summarizer.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/fixtures/rails_1x.log lib/request_log_analyzer/file_format/delayed_job.rb lib/request_log_analyzer/output.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/file_format/mysql.rb spec/unit/file_format/mysql_format_spec.rb request-log-analyzer.gemspec spec/fixtures/apache_common.log lib/request_log_analyzer/file_format/rails.rb Rakefile spec/unit/mailer_spec.rb lib/request_log_analyzer/tracker/hourly_spread.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb lib/request_log_analyzer/tracker/timespan.rb spec/unit/controller/log_processor_spec.rb spec/fixtures/sinatra.log spec/unit/filter/field_filter_spec.rb spec/fixtures/merb_prefixed.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/filter/timespan.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/mailer.rb lib/cli/tools.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/timespan_tracker_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/controller.rb spec/unit/tracker/tracker_api_spec.rb lib/request_log_analyzer/source/database_loader.rb spec/fixtures/merb.log LICENSE lib/request_log_analyzer/file_format.rb tasks/request_log_analyzer.rake spec/unit/database/database_spec.rb spec/fixtures/decompression.log.zip spec/fixtures/decompression.tgz bin/request-log-analyzer lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/source.rb spec/fixtures/mysql_slow_query.log lib/request_log_analyzer/source/log_parser.rb lib/request_log_analyzer/aggregator/echo.rb)
+  s.test_files = %w(spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/database/base_class_spec.rb spec/unit/file_format/rack_format_spec.rb spec/unit/filter/filter_spec.rb spec/unit/database/connection_spec.rb spec/integration/scout_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/format_autodetection_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/mailer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/munin_plugins_rails_spec.rb spec/unit/file_format/delayed_job_format_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/file_format/rails_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/request_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/filter/anonymize_filter_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/common_regular_expressions_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/file_format/mysql_format_spec.rb spec/unit/mailer_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/tracker/numeric_value_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/controller/controller_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/database/database_spec.rb)
 end

data/spec/unit/file_format/delayed_job_format_spec.rb

@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::FileFormat::DelayedJob do
+
+  it "should be a valid file format" do
+    RequestLogAnalyzer::FileFormat.load(:delayed_job).should be_valid
+  end
+
+  describe '#parse_line' do
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:delayed_job)
+    end
+
+    it "should parse a :job_lock line correctly" do
+      line = "* [JOB] acquiring lock on BackgroundJob::ThumbnailSaver"
+      @file_format.should parse_line(line).as(:job_lock).and_capture(:job => 'BackgroundJob::ThumbnailSaver')
+    end
+
+    it "should parse a :job_completed line correctly" do
+      line = '* [JOB] BackgroundJob::ThumbnailSaver completed after 0.7932'
+      @file_format.should parse_line(line).as(:job_completed).and_capture(
+        :duration => 0.7932, :completed_job => 'BackgroundJob::ThumbnailSaver')
+    end
+    
+    it "should pase a :job_failed line correctly" do
+      line = "* [JOB] BackgroundJob::ThumbnailSaver failed with ActiveRecord::RecordNotFound: Couldn't find Design with ID=20413443 - 1 failed attempts"
+      @file_format.should parse_line(line).as(:job_failed).and_capture(:attempts => 1,
+        :failed_job => 'BackgroundJob::ThumbnailSaver', :exception => 'ActiveRecord::RecordNotFound')
+    end
+    
+    it "" do
+      line = "* [JOB] failed to acquire exclusive lock for BackgroundJob::ThumbnailSaver"
+      @file_format.should parse_line(line).as(:job_lock_failed).and_capture(:locked_job => 'BackgroundJob::ThumbnailSaver')
+    end
+    
+    # it "should pase a :batch_completed line correctly" do
+    #   line = '1 jobs processed at 1.0834 j/s, 0 failed ...'
+    #   @file_format.should parse_line(line).as(:batch_completed).and_capture(
+    #     :mean_duration => 0.7932, :total_amount => 1, :failed_amount => 0) 
+    # end
+
+  end
+  
+  describe '#parse_io' do
+    before(:each) do
+      @log_parser = RequestLogAnalyzer::Source::LogParser.new(RequestLogAnalyzer::FileFormat.load(:delayed_job))
+    end
+    
+    it "should parse a batch of completed jobs without warnings" do
+      fragment = "* [JOB] acquiring lock on BackgroundJob::ThumbnailSaver
+                  * [JOB] BackgroundJob::ThumbnailSaver completed after 0.9114
+                  * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver
+                  * [JOB] BackgroundJob::ThumbnailSaver completed after 0.9110
+                  2 jobs processed at 1.0832 j/s, 0 failed ..."
+
+      request_counter.should_receive(:hit!).exactly(2).times
+      @log_parser.should_not_receive(:warn)
+
+      @log_parser.parse_io(StringIO.new(fragment)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::DelayedJob::Request)
+      end
+    end
+    
+    it "should parse a batch with a failed job without warnings" do
+      fragment = "* [JOB] acquiring lock on BackgroundJob::ThumbnailSaver
+                  * [JOB] BackgroundJob::ThumbnailSaver completed after 1.0627
+                  * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver
+                  * [JOB] BackgroundJob::ThumbnailSaver failed with ActiveRecord::RecordNotFound: Couldn't find Design with ID=20413443 - 3 failed attempts
+                  Couldn't find Design with ID=20413443
+                  * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver
+                  * [JOB] failed to acquire exclusive lock for BackgroundJob::ThumbnailSaver
+                  2 jobs processed at 1.4707 j/s, 1 failed ..."
+
+      request_counter.should_receive(:hit!).exactly(3).times
+      @log_parser.should_not_receive(:warn)
+
+      @log_parser.parse_io(StringIO.new(fragment)) do |request|
+        request_counter.hit! if request.kind_of?(RequestLogAnalyzer::FileFormat::DelayedJob::Request)
+      end
+    end
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.5.3
+  version: 1.5.4
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-14 00:00:00 +01:00
+date: 2009-12-17 00:00:00 +01:00
 default_executable: request-log-analyzer
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -93,6 +93,7 @@ files:
 - spec/fixtures/syslog_1x.log
 - spec/integration/munin_plugins_rails_spec.rb
 - spec/lib/helpers.rb
+- spec/unit/file_format/delayed_job_format_spec.rb
 - spec/fixtures/rails.db
 - spec/unit/source/log_parser_spec.rb
 - lib/request_log_analyzer/aggregator/database_inserter.rb
@@ -119,11 +120,12 @@ files:
 - lib/request_log_analyzer/database/warning.rb
 - spec/unit/filter/anonymize_filter_spec.rb
 - spec/fixtures/apache_combined.log
+- spec/unit/tracker/duration_tracker_spec.rb
 - spec/fixtures/rails_22_cached.log
 - lib/request_log_analyzer/aggregator/summarizer.rb
-- spec/unit/tracker/duration_tracker_spec.rb
 - spec/unit/file_format/common_regular_expressions_spec.rb
 - spec/fixtures/rails_1x.log
+- lib/request_log_analyzer/file_format/delayed_job.rb
 - lib/request_log_analyzer/output.rb
 - spec/unit/file_format/line_definition_spec.rb
 - lib/request_log_analyzer/file_format/mysql.rb
@@ -215,6 +217,7 @@ test_files:
 - spec/unit/tracker/frequency_tracker_spec.rb
 - spec/unit/file_format/apache_format_spec.rb
 - spec/integration/munin_plugins_rails_spec.rb
+- spec/unit/file_format/delayed_job_format_spec.rb
 - spec/unit/source/log_parser_spec.rb
 - spec/unit/file_format/rails_format_spec.rb
 - spec/integration/command_line_usage_spec.rb