request-log-analyzer 1.0.2 → 1.6.3

data/lib/request_log_analyzer/file_format/rails.rb

@@ -1,90 +1,176 @@
-class RequestLogAnalyzer::FileFormat::Rails < RequestLogAnalyzer::FileFormat
-  
-  # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
-  line_definition :processing do |line|
-    line.header = true # this line is the first log line for a request 
-    line.teaser = /Processing /
-    line.regexp = /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/
-    line.captures << { :name => :controller, :type  => :string } \
-                  << { :name => :action,     :type  => :string } \
-                  << { :name => :format,     :type  => :string } \
-                  << { :name => :ip,         :type  => :string, :anonymize => :ip } \
-                  << { :name => :timestamp,  :type  => :timestamp, :anonymize => :slightly } \
-                  << { :name => :method,     :type  => :string }
-  end
-
-  # Filter chain halted as [#<ActionController::Caching::Actions::ActionCacheFilter:0x2a999ad620 @check=nil, @options={:store_options=>{}, :layout=>nil, :cache_path=>#<Proc:0x0000002a999b8890@/app/controllers/cached_controller.rb:8>}>] rendered_or_redirected.    
-  line_definition :cache_hit do |line|
-    line.regexp = /Filter chain halted as \[\#<ActionController::Caching::Actions::ActionCacheFilter:.+>\] rendered_or_redirected/
-  end
-  
-  # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
-  line_definition :failed do |line|
-    line.footer = true
-    line.regexp = /((?:[A-Z]\w+\:\:)*[A-Z]\w+) \((.*)\)(?: on line #(\d+) of .+)?\:(.*)/
-    line.captures << { :name => :error,       :type => :string } \
-                  << { :name => :message,     :type => :string } \
-                  << { :name => :line,        :type => :integer } \
-                  << { :name => :file,        :type => :string } \
-                  << { :name => :stack_trace, :type => :string, :anonymize => true }
-  end
+module RequestLogAnalyzer::FileFormat
 
+  # Default FileFormat class for Rails logs.
+  #
+  # Instances will be created dynamically based on the lines you want it to parse. You can
+  # specify what lines should be included in the parser by providing a list to the create
+  # method as first argument.
+  class Rails < Base
 
-  # Rails < 2.1 completed line example
-  # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
-  RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
+    extend CommonRegularExpressions
 
-  # Rails > 2.1 completed line example
-  # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
-  RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
+    # Creates a Rails FileFormat instance.
+    #
+    # The lines that will be parsed can be defined by the argument to this function, 
+    # which should be an array of line names, or a list of line names as comma separated
+    # string. The resulting report depends on the lines that will be parsed. You can 
+    # also provide s string that describes a common set of lines, like "production",
+    # "development" or "production".
+    def self.create(lines = 'production')
+      definitions_hash = line_definer.line_definitions.clone
+      
+      lines = lines.to_s.split(',') if lines.kind_of?(String)
+      lines = [lines.to_s]          if lines.kind_of?(Symbol)
+      
+      lines.each do |line|
+        line = line.to_sym
+        if LINE_COLLECTIONS.has_key?(line)
+          LINE_COLLECTIONS[line].each { |l| definitions_hash[l] = LINE_DEFINITIONS[l] }
+        elsif LINE_DEFINITIONS.has_key?(line)
+          definitions_hash[line] = LINE_DEFINITIONS[line]
+        else
+          raise "Unrecognized Rails log line name: #{line.inspect}!"
+        end
+      end
 
-  # The completed line uses a kind of hack to ensure that both old style logs and new style logs 
-  # are both parsed by the same regular expression. The format in Rails 2.2 was slightly changed,
-  # but the line contains exactly the same information.
-  line_definition :completed do |line|
-    
-    line.footer = true
-    line.teaser = /Completed in /
-    line.regexp = Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})")
+      return self.new(definitions_hash, report_trackers(definitions_hash))
+    end
     
-    line.captures << { :name => :duration, :type => :sec,    :anonymize => :slightly } \
-                  << { :name => :view,     :type => :sec,    :anonymize => :slightly } \
-                  << { :name => :db,       :type => :sec,    :anonymize => :slightly } \
-                  << { :name => :status,   :type => :integer } \
-                  << { :name => :url,      :type => :string, :anonymize => :url } # Old variant 
-                  
-    line.captures << { :name => :duration, :type => :msec,   :anonymize => :slightly } \
-                  << { :name => :view,     :type => :msec,   :anonymize => :slightly } \
-                  << { :name => :db,       :type => :msec,   :anonymize => :slightly } \
-                  << { :name => :status,   :type => :integer} \
-                  << { :name => :url,      :type => :string, :anonymize => :url }  # 2.2 variant 
-  end
+    # Creates trackers based on the specified line definitions.
+    #
+    # The more lines that will be parsed, the more information will appear in the report.
+    def self.report_trackers(lines)
+      analyze = RequestLogAnalyzer::Aggregator::Summarizer::Definer.new
+      
+      analyze.timespan
+      analyze.hourly_spread
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Most requested'
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      
+      if lines.has_key?(:cache_hit)
+        analyze.frequency(:category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, 
+              :title => 'Rails action cache hits')
+      end
+      
+      analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
+      analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+      analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
+        :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
+      
+      if lines.has_key?(:failure)
+        analyze.frequency :error, :title => 'Failed requests', :line_type => :failure
+      end
 
+      if lines.has_key?(:rendered)
+        analyze.duration :render_duration, :category => :render_file, :multiple => true, :title => 'Partial rendering duration'
+      end
 
+      if lines.has_key?(:query_executed)
+        analyze.duration :query_duration, :category => :query_sql, :multiple => true, :title => 'Query duration'
+      end
+      
+      return analyze.trackers + report_definer.trackers
+    end
 
-  REQUEST_CATEGORIZER = Proc.new do |request|
-    format = request[:format] || 'html'
-    "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-  end
+    # Rails < 2.1 completed line example
+    # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+    RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
 
-  report do |analyze|
-    analyze.timespan :line_type => :processing
-    analyze.category :category => REQUEST_CATEGORIZER, :title => 'Top 20 hits', :amount => 20, :line_type => :processing
-    analyze.category :method, :title => 'HTTP methods'
-    analyze.category :status, :title => 'HTTP statuses returned'
-    analyze.category :category => lambda { |request| request =~ :cache_hit ? 'Cache hit' : 'No hit' }, :title => 'Rails action cache hits'
-    
-    analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
-    analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
-    analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
-    
-    analyze.category :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)', 
-            :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }, :amount => 20
-            
-    analyze.hourly_spread :line_type => :processing
-    analyze.category :error, :title => 'Failed requests', :line_type => :failed, :amount => 20
-  end
+    # Rails > 2.1 completed line example
+    # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
+    RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
+
+    # A hash of definitions for all common lines in Rails logs.
+    LINE_DEFINITIONS = {
+      :processing => RequestLogAnalyzer::LineDefinition.new(:processing, :header => true,
+            :teaser   => /Processing /,
+            :regexp   => /Processing ((?:\w+::)*\w+)#(\w+)(?: to (\w+))? \(for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})\) \[([A-Z]+)\]/,
+            :captures => [{ :name => :controller, :type  => :string },
+                          { :name => :action,     :type  => :string },
+                          { :name => :format,     :type  => :string, :default => 'html' },
+                          { :name => :ip,         :type  => :string },
+                          { :name => :timestamp,  :type  => :timestamp },
+                          { :name => :method,     :type  => :string }]),
 
+      :completed => RequestLogAnalyzer::LineDefinition.new(:completed, :footer => true,
+            :teaser   => /Completed in /,
+            :regexp   => Regexp.union(RAILS_21_COMPLETED, RAILS_22_COMPLETED),
+            :captures => [{ :name => :duration, :type => :duration, :unit => :sec },   # First old variant capture
+                          { :name => :view,     :type => :duration, :unit => :sec },
+                          { :name => :db,       :type => :duration, :unit => :sec },
+                          { :name => :status,   :type => :integer },
+                          { :name => :url,      :type => :string },                    # Last old variant capture
+                          { :name => :duration, :type => :duration, :unit => :msec },  # First new variant capture
+                          { :name => :view,     :type => :duration, :unit => :msec },
+                          { :name => :db,       :type => :duration, :unit => :msec },
+                          { :name => :status,   :type => :integer },
+                          { :name => :url,      :type => :string }]),                  # Last new variant capture
 
+      :failure => RequestLogAnalyzer::LineDefinition.new(:failure, :footer => true,
+          :teaser   => /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:/,
+          :regexp   => /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:\s*$/,
+          :captures => [{ :name => :error,       :type => :string },
+                        { :name => :message,     :type => :string },
+                        { :name => :line,        :type => :integer },
+                        { :name => :file,        :type => :string }]),
 
-end 
+      :cache_hit => RequestLogAnalyzer::LineDefinition.new(:cache_hit,
+          :regexp => /Filter chain halted as \[\#<ActionController::Filters::AroundFilter.*\@method=.*(?:Caching::Actions::ActionCacheFilter|action_controller\/caching\/actions\.rb).*\] did_not_yield/),
+
+      :parameters => RequestLogAnalyzer::LineDefinition.new(:parameters,
+          :teaser   => /  Parameters:/,
+          :regexp   => /  Parameters:\s+(\{.*\})/,
+          :captures => [{ :name => :params, :type => :eval }]),
+
+      :rendered => RequestLogAnalyzer::LineDefinition.new(:rendered,
+          :teaser   => /Rendered /,
+          :regexp   => /Rendered (\w+(?:\/\w+)+) \((\d+\.\d+)ms\)/,
+          :captures => [{ :name => :render_file,     :type  => :string },
+                        { :name => :render_duration, :type  => :duration, :unit => :msec }]),
+
+      :query_executed => RequestLogAnalyzer::LineDefinition.new(:query_executed,
+          :regexp   => /\s+(?:\e\[4;36;1m)?((?:\w+::)*\w+) Load \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0;1m)?([^\e]+) ?(?:\e\[0m)?/,
+          :captures => [{ :name => :query_class,    :type  => :string },
+                        { :name => :query_duration, :type  => :duration, :unit => :msec },
+                        { :name => :query_sql,      :type  => :sql }]),
+
+      :query_cached => RequestLogAnalyzer::LineDefinition.new(:query_cached,
+          :regexp   => /\s+(?:\e\[4;35;1m)?CACHE \((\d+\.\d+)ms\)(?:\e\[0m)?\s+(?:\e\[0m)?([^\e]+) ?(?:\e\[0m)?/,
+          :captures => [{ :name => :cached_duration, :type  => :duration, :unit => :msec },
+                        { :name => :cached_sql,      :type  => :sql }])
+    }
+
+    # Definitions of common combinations of lines that can be parsed
+    LINE_COLLECTIONS = {
+      :minimal     => [:processing, :completed],
+      :production  => [:processing, :completed, :failure, :cache_hit],
+      :development => [:processing, :completed, :failure, :rendered, :query_executed, :query_cached],
+      :all         => LINE_DEFINITIONS.keys
+    }
+
+
+    # Simple function to categorize Rails requests using controller/actions/format and method.
+    REQUEST_CATEGORIZER = Proc.new do |request|
+      "#{request[:controller]}##{request[:action]}.#{request[:format]} [#{request[:method]}]"
+    end
+
+    # Define a custom Request class for the Rails file format to speed up timestamp handling
+    # and to ensure that a format is always set.
+    class Request < RequestLogAnalyzer::Request
+
+      # Do not use DateTime.parse
+      def convert_timestamp(value, definition)
+        value.gsub(/[^0-9]/, '')[0...14].to_i
+      end
+
+      # Sanitizes SQL queries so that they can be grouped
+      def convert_sql(sql, definition)
+        sql.gsub(/\b\d+\b/, ':int').gsub(/`([^`]+)`/, '\1').gsub(/'[^']*'/, ':string').rstrip
+      end
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -0,0 +1,86 @@
+module RequestLogAnalyzer::FileFormat
+
+  # Default FileFormat class for Rails 3 logs.
+  #
+  # For now, this is just a basic implementation. It will probaby change after 
+  # Rails 3 final has been released.
+  class Rails3 < Base
+
+    extend CommonRegularExpressions
+
+    # Started GET "/queries" for 127.0.0.1 at 2010-02-25 16:15:18
+    line_definition :started do |line|
+      line.header = true
+      line.teaser = /Started /
+      line.regexp = /Started ([A-Z]+) "([^"]+)" for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})/
+      
+      line.capture(:method)
+      line.capture(:path)
+      line.capture(:ip)
+      line.capture(:timestamp).as(:timestamp)
+    end
+    
+    # Processing by QueriesController#index as HTML
+    line_definition :processing do |line|
+      line.teaser = /Processing by /
+      line.regexp = /Processing by (\w+)\#(\w+) as (\w+)/
+      
+      line.capture(:controller)
+      line.capture(:action)
+      line.capture(:format)
+    end
+    
+    # Completed in 9ms (Views: 4.9ms | ActiveRecord: 0.5ms) with 200
+    line_definition :completed do |line|
+      line.footer = true
+      line.teaser = /Completed /
+      line.regexp = /Completed (\d+) .* in (\d+)ms \([^\)]*\)/
+
+      line.capture(:status).as(:integer)
+      line.capture(:duration).as(:duration, :unit => :msec)
+    end
+    
+    # ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:
+    line_definition :failure do |line|
+      line.footer = true
+      line.regexp = /((?:[A-Z]\w*[a-z]\w+\:\:)*[A-Z]\w*[a-z]\w+) \((.*)\)(?: on line #(\d+) of (.+))?\:\s*$/
+
+      line.capture(:error)
+      line.capture(:message)
+      line.capture(:line).as(:integer)
+      line.capture(:file)
+    end
+    
+    # # Not parsed at the moment:
+    #  SQL (0.2ms)  SET SQL_AUTO_IS_NULL=0
+    #  Query Load (0.4ms)  SELECT `queries`.* FROM `queries`
+    # Rendered collection (0.0ms)
+    # Rendered queries/index.html.erb (0.6ms)
+    
+    REQUEST_CATEGORIZER = lambda { |request| "#{request[:controller]}##{request[:action]}.#{request[:format]}" }
+    
+    report do |analyze|
+      
+      analyze.timespan
+      analyze.hourly_spread
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Most requested'
+      analyze.frequency :method, :title => 'HTTP methods'
+      analyze.frequency :status, :title => 'HTTP statuses returned'
+      
+      analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
+      # analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+      # analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
+      
+      analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
+        :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
+    end
+    
+    class Request < RequestLogAnalyzer::Request
+      def convert_timestamp(value, defintion)
+        value.gsub(/[^0-9]/, '')[0...14].to_i
+      end
+    end
+
+  end
+end

data/lib/request_log_analyzer/file_format/rails_development.rb

@@ -0,0 +1,12 @@
+module RequestLogAnalyzer::FileFormat
+
+  # The RailsDevelopment FileFormat is an extention to the default Rails file format. It includes
+  # all lines of the normal Rails file format, but parses SQL queries and partial rendering lines
+  # as well.
+  class RailsDevelopment < Rails
+    def self.create
+      # puts 'DEPRECATED: use --rails-format development instead!'
+      super('development')
+    end
+  end
+end