wvanbergen-request-log-analyzer 0.1.2 → 0.2.2

data/README

@@ -1,9 +1,8 @@
 Request log analyzer
 --------------------------------
 
-This is a simple command line tool to analyze request log files. At this moment,
-it only supports Rails log files, but Merb log files are planned to be supported
-as well. Its purpose is to find what actions are best candidates for optimization.
+This is a simple command line tool to analyze request log files of both Rails and
+Merb. Its purpose is to find what actions are best candidates for optimization.
 
 This tool will parse all requests in the logfile and aggregate the
 information. Once it is finished parsing the log file, it will show the
@@ -46,6 +45,7 @@ Successfully analyzed 58908 requests from log file
 Timestamp first request: 2008-07-13T06:25:58+00:00
 Timestamp last request:  2008-07-20T06:18:53+00:00
 Total time analyzed: 7 days
+Methods: DELETE (1%), GET (50%), POST (22%), PUT (25%).
 
 Top 10 most requested actions
 ========================================================================
@@ -92,7 +92,7 @@ Top 10 worst DB offenders - mean time
 /overview/:date/                                  :      0.467s [19359 requests]
 ........
 
-Mongrel process blockers (> 1.0 seconds) - frequency
+Mongrel process blockers (> 1.0 seconds)
 ========================================================================
 /overview/:date/                                  :   7494.233s [3144 requests]
 /overview/                                        :   8320.293s [1549 requests]
@@ -117,3 +117,16 @@ Requests graph - per hour
         17:00 - 1598                 : XXXX
         18:00 - 792                  : XX
         ........
+
+Errors
+========================================================================
+ArgumentError: [237 requests]
+ -> invalid date
+StaleObjectError: [28 requests]
+ -> Attempted to update a stale object
+RuntimeError: [3 requests]
+ -> Cannot destroy rule before it was created
+StatementError: [2 requests]
+ -> Mysql::Error: Deadlock found when trying to get lock; try restarting transaction
+NoMethodError: [1 requests]
+ -> undefined method `code' for nil:NilClass

data/Rakefile

@@ -2,14 +2,14 @@ require 'rubygems'
 
 load 'test/tasks.rake'
  
-desc 'Default: run unit tests.'
+desc 'Default: run unit tests for request-log-analyzer.'
 task :default => :test
 
 
 namespace :gem do
 
-  desc "Sets the version and date of the gem"
-  task :version do
+  desc "Sets the version and date of the gem. Requires the VERSION environment variable."
+  task :version => [:manifest] do
     
     require 'date'
     
@@ -24,6 +24,7 @@ namespace :gem do
     File.open(spec_file, 'w') { |f| f << spec }
   end
   
+  desc "Creates a git tag for the provided VERSION"
   task :tag => [:version] do
     
     new_version = ENV['VERSION']

data/TODO

@@ -2,9 +2,6 @@ TODO items for Rails-log-analyzer
 =================================
 Contact willem AT vanbergen DOT org if you want to help out with the development.
 
-Summarizer:
-- Look at request types (GET / POST)
-
 Database:
 - Add query functionality for the resulting database file (interactive reports?)
 - Link request processing line to request completed line
@@ -17,4 +14,5 @@ Rails integration:
 General:
 - Add useful rake tasks
 - Add more tests
+- Fix multiple file handling
 - World domination

data/bin/request-log-analyzer

@@ -1,14 +1,18 @@
 #!/usr/bin/ruby
 require File.dirname(__FILE__) + '/../lib/command_line/arguments'
+require File.dirname(__FILE__) + '/../lib/base/log_parser'
+require File.dirname(__FILE__) + '/../lib/base/summarizer'
 require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
 require File.dirname(__FILE__) + '/../lib/rails_analyzer/summarizer'
+require File.dirname(__FILE__) + '/../lib/merb_analyzer/log_parser'
+require File.dirname(__FILE__) + '/../lib/merb_analyzer/summarizer'
 require File.dirname(__FILE__) + '/../lib/bashcolorizer'
 require File.dirname(__FILE__) + '/../lib/ruby-progressbar/progressbar.rb'
 
 puts "Request log analyzer, by Willem van Bergen and Bart ten Brinke\n\n"
 
 # Substitutes variable elements in a url (like the id field) with a fixed string (like ":id")
-# This is used to aggregate simular requests.
+# This is used to aggregate simular requests. 
 # <tt>request</tt> The request to evaluate.
 # Returns uniformed url string.
 # Raises on mailformed request.
@@ -17,9 +21,9 @@ def request_hasher(request)
     url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
     url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
 
-    url.gsub!(/\/\d+-\d+-\d+/, '/:date') # Combine all (year-month-day) queries
-    url.gsub!(/\/\d+-\d+/, '/:month') # Combine all date (year-month) queries
-    url.gsub!(/\/\d+/, '/:id') # replace identifiers in URLs
+    url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date') # Combine all (year-month-day) queries
+    url.gsub!(/\/\d+-\d+(\/|$)/, '/:month') # Combine all date (year-month) queries
+    url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs
         
     return url
   elsif request[:controller] && request[:action]
@@ -48,6 +52,7 @@ begin
     command_line.switch(:guess_database_time, :g)
     command_line.switch(:fast, :f)
     command_line.switch(:colorize, :z)
+    command_line.switch(:merb, :m)
     command_line.flag(:output, :alias => :o)
     command_line.flag(:amount, :alias => :c)
     command_line.required_files = 1
@@ -60,15 +65,30 @@ rescue CommandLine::Error => e
   exit(0) 
 end
 
-$summarizer = RailsAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
-$summarizer.blocker_duration = 1.0
 
-line_types = $arguments[:fast] ? [:completed] : [:started, :completed, :failed]
+if $arguments[:merb]
+  $summarizer = MerbAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
+else
+  $summarizer = RailsAnalyzer::Summarizer.new(:calculate_database => $arguments[:guess_database_time])
+end
+
+if $arguments[:fast]
+  line_types = [:completed]
+elsif $arguments[:merb]
+  line_types = MerbAnalyzer::LogParser::LOG_LINES.keys
+else
+  line_types = RailsAnalyzer::LogParser::LOG_LINES.keys
+end
 
 # Walk through al the files given via the arguments.
 $arguments.files.each do |log_file|
   puts "Processing #{line_types.join(', ')} log lines from #{log_file}..."
-  parser = RailsAnalyzer::LogParser.new(log_file)
+
+  if $arguments[:merb]
+    parser = MerbAnalyzer::LogParser.new(log_file)
+  else
+    parser = RailsAnalyzer::LogParser.new(log_file)
+  end
   
   # add progress bar 
   unless $arguments[:fast]

data/bin/request-log-database

@@ -1,6 +1,8 @@
 #!/usr/bin/ruby
 
 require File.dirname(__FILE__) + '/../lib/command_line/arguments'
+require File.dirname(__FILE__) + '/../lib/base/log_parser'
+require File.dirname(__FILE__) + '/../lib/base/record_inserter'
 require File.dirname(__FILE__) + '/../lib/rails_analyzer/log_parser'
 require File.dirname(__FILE__) + '/../lib/rails_analyzer/record_inserter'
 require File.dirname(__FILE__) + '/../lib/bashcolorizer'

data/lib/base/log_parser.rb

@@ -0,0 +1,78 @@
+module Base
+  # Parse a log file
+  class LogParser 
+
+    LOG_LINES = {}
+  
+    # LogParser initializer
+    # <tt>file</tt> The fileobject this LogParser wil operate on.
+    def initialize(file, options = {})
+      @file_name = file
+      @options = options
+      @file_size = File.size(@file_name)
+      
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+  
+    def progress(&block)
+      @progress_handler = block
+    end
+
+    # Output a warning
+    # <tt>message</tt> The warning message (object)
+    def warn(message)
+      puts " -> " + message.to_s
+    end  
+
+    def convert_value(value, type) 
+      return case type
+      when :string;    value.to_s
+      when :int;       value.to_i
+      when :sec;       value.to_f
+      when :msec;      value.to_f / 1000
+      when :timestamp; value.to_s # TODO: fix me?
+      else
+        warn("Unkwn type encountered: #{type}")
+        value
+      end
+    end
+
+    # Finds a log line and then parses the information in the line.
+    # Yields a hash containing the information found. 
+    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
+    # Yeilds a Hash when it encounters a chunk of information.
+    def each(*line_types, &block)
+      log_lines_hash = self.class::LOG_LINES
+
+      
+      # parse everything by default 
+      line_types = log_lines_hash.keys if line_types.empty?
+
+      File.open(@file_name) do |file|
+      
+        file.each_line do |line|
+        
+          @progress_handler.call(file.pos, @file_size) if @progress_handler
+          
+          line_types.each do |line_type|
+            if log_lines_hash[line_type][:teaser] =~ line
+              if log_lines_hash[line_type][:regexp] =~ line
+                
+                captures = $~.captures
+                request = { :type => line_type, :line => file.lineno }
+                log_lines_hash[line_type][:params].each_with_index do |paramhash, index|              
+                  paramhash.each { |key, type| request[key] = convert_value(captures[index], type) } unless captures[index].nil?
+                end
+                
+                yield(request) if block_given?
+              else
+                warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
+              end
+            end            
+          end
+        end
+        @progress_handler.call(:finished, @file_size) if @progress_handler
+      end      
+    end
+  end
+end

data/lib/base/record_inserter.rb

@@ -0,0 +1,139 @@
+require 'rubygems'
+require 'sqlite3'
+
+module Base
+  
+  # Set of functions that can be used to easily log requests into a SQLite3 Database.
+  class RecordInserter
+    
+    attr_reader :database
+    attr_reader :current_request
+    attr_reader :warning_count
+    
+    # Initializer
+    # <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
+    def initialize(db_file, options = {})
+      @database = SQLite3::Database.new(db_file)
+      @insert_statements = nil
+      @warning_count = 0
+      create_tables_if_needed!
+
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+        
+    # Calculate the database durations of the requests currenty in the database.
+    # Used if a logfile does contain any database durations.
+    def calculate_db_durations!
+      @database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
+    end
+    
+    # Insert a batch of loglines into the database.
+    # Function prepares insert statements, yeilds and then closes and commits.
+    def insert_batch(&block)
+      @database.transaction
+      prepare_statements!
+      block.call(self)
+      close_prepared_statements!
+      @database.commit
+    rescue Exception => e
+      puts e.message
+      @database.rollback
+    end
+    
+    def insert_warning(line, warning)
+      @database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
+      @warning_count += 1
+    end
+        
+    # Insert a request into the database.
+    # def insert(request, close_statements = false)
+    #   raise 'No insert defined for this log file type'
+    # end
+    
+    # Insert a batch of files into the database.
+    # <tt>db_file</tt> The filename of the database file to use.
+    # Returns the created database.
+    def self.insert_batch_into(db_file, options = {}, &block)
+      db = self.new(db_file)
+      db.insert_batch(&block)
+      return db
+    end    
+    
+    def count(type)
+      @database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
+    end
+        
+    protected
+    
+    # Prepare insert statements.
+    def prepare_statements!
+      @insert_statements = {
+        :started => @database.prepare("
+            INSERT INTO started_requests ( line,  timestamp,  ip,  method,  controller,  action) 
+                                  VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
+                                  
+        :failed => @database.prepare("
+            INSERT INTO failed_requests ( line,  exception_string,  stack_trace,  error)
+                                 VALUES (:line, :exception_string, :stack_trace, :error)"),
+                                 
+        :completed => @database.prepare("
+            INSERT INTO completed_requests ( line,  url,  status,  duration,  rendering_time,  database_time)
+                                    VALUES (:line, :url, :status, :duration, :rendering, :db)")
+      }
+    end
+    
+    # Close all prepared statments
+    def close_prepared_statements!
+      @insert_statements.each { |key, stmt| stmt.close }
+    end
+
+    # Create the needed database tables if they don't exist.
+    def create_tables_if_needed!
+      
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS started_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,
+          timestamp DATETIME NOT NULL, 
+          controller VARCHAR(255) NOT NULL, 
+          action VARCHAR(255) NOT NULL,
+          method VARCHAR(6) NOT NULL,          
+          ip VARCHAR(6) NOT NULL
+        )
+      ");
+
+      @database.execute("
+          CREATE TABLE IF NOT EXISTS failed_requests (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            line INTEGER NOT NULL,            
+            started_request_id INTEGER,        
+            error VARCHAR(255),    
+            exception_string VARCHAR(255),
+            stack_trace TEXT
+          )      
+      ");
+
+      @database.execute("
+        CREATE TABLE IF NOT EXISTS completed_requests (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,          
+          started_request_id INTEGER,
+          url VARCHAR(255) NOT NULL,
+          hashed_url VARCHAR(255),
+          status INTEGER NOT NULL,
+          duration FLOAT,
+          rendering_time FLOAT,
+          database_time FLOAT
+        )
+      ");    
+      
+      @database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          line INTEGER NOT NULL,       
+          warning VARCHAR(255) NOT NULL
+        )
+      ");
+    end
+
+  end
+end

data/lib/base/summarizer.rb

@@ -0,0 +1,71 @@
+module Base
+  
+  # Functions to summarize an array of requets.
+  # Can calculate request counts, duratations, mean times etc. of all the requests given.
+  class Summarizer
+    attr_reader :actions
+    attr_reader :errors
+    attr_reader :request_count
+    attr_reader :request_time_graph
+    attr_reader :first_request_at
+    attr_reader :last_request_at
+    attr_reader :methods
+
+    attr_accessor :blocker_duration
+    DEFAULT_BLOCKER_DURATION = 1.0
+   
+    # Initializer. Sets global variables
+    # Options
+    def initialize(options = {})
+      @actions  = {}
+      @blockers = {}
+      @errors   = {}
+      @request_count = 0
+      @blocker_duration = DEFAULT_BLOCKER_DURATION
+      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+      @methods = {:GET => 0, :POST => 0, :PUT => 0, :DELETE => 0, :unknown => 0}
+      
+      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
+    end
+   
+    # Check if any of the request parsed had a timestamp.
+    def has_timestamps?
+      @first_request_at
+    end
+   
+    # Calculate the duration of a request
+    # Returns a DateTime object if possible, 0 otherwise.
+    def duration
+      (@last_request_at && @first_request_at) ? (DateTime.parse(@last_request_at) - DateTime.parse(@first_request_at)).ceil : 0
+    end
+    
+    # Check if the request time graph usable data.
+    def request_time_graph?
+      @request_time_graph.uniq != [0] && duration > 0
+    end
+
+    # Return a list of requests sorted on a specific action field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default nil).
+    def sort_actions_by(field, min_count = nil)
+      actions = min_count.nil? ? @actions.to_a : @actions.delete_if { |k, v| v[:count] < min_count}.to_a
+      actions.sort { |a, b| (a[1][field.to_sym] <=> b[1][field.to_sym]) }
+    end
+
+    # Returns a list of request blockers sorted by a specific field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
+    def sort_blockers_by(field, min_count = @blocker_duration)
+      blockers = min_count.nil? ? @blockers.to_a : @blockers.delete_if { |k, v| v[:count] < min_count}.to_a
+      blockers.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
+    end
+
+    # Returns a list of request blockers sorted by a specific field
+    # <tt>field</tt> The action field to sort by.
+    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
+    def sort_errors_by(field, min_count = nil)
+      errors = min_count.nil? ? @errors.to_a : @errors.delete_if { |k, v| v[:count] < min_count}.to_a
+      errors.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
+    end
+  end
+end 

data/lib/merb_analyzer/log_parser.rb

@@ -0,0 +1,26 @@
+module MerbAnalyzer
+
+  class LogParser < Base::LogParser
+    LOG_LINES = {
+      # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+      :started => {
+        :teaser => /Started/,
+        :regexp => /Started request handling\:\ (.+)/,
+        :params => [{:timestamp => :timestamp}]
+      },
+      # ~ Params: {"action"=>"create", "controller"=>"session"}
+      # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"d}
+      :params => {
+        :teaser => /Params/,
+        :regexp => /Params\:\ \{(.+)\}/,
+        :params => [{:raw_hash => :string}]
+      },
+      # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
+      :completed => {
+        :teaser => /\{:dispatch_time/,
+        :regexp => /\{\:dispatch_time=>(.+), (?:\:after_filters_time=>(.+), )?(?:\:before_filters_time=>(.+), )?\:action_time=>(.+)\}/,
+        :params => [ {:dispatch_time => :sec}, {:after_filters_time => :sec}, {:before_filters_time => :sec}, {:action_time => :sec} ]
+      }
+    }
+  end
+end

data/lib/merb_analyzer/summarizer.rb

@@ -0,0 +1,61 @@
+require 'date'
+module MerbAnalyzer
+  
+  # Functions to summarize an array of requets.
+  # Can calculate request counts, duratations, mean times etc. of all the requests given.
+  class Summarizer < Base::Summarizer
+
+    # Initializer. Sets global variables
+    def initialize_hook(options = {})
+      @hash_cache = nil
+    end
+
+    # Parse a request string into a hash containing all keys found in the string.
+    # Yields the hash found to the block operator.
+    # <tt>request</tt> The request string to parse.
+    # <tt>&block</tt> Block operator
+    def group(request, &block)
+      request[:duration] ||= 0
+      
+      case request[:type]   
+        when :started 
+          @first_request_at ||= request[:timestamp] # assume time-based order of file
+          @last_request_at  = request[:timestamp]   # assume time-based order of file
+          @request_time_graph[request[:timestamp][11..12].to_i] +=1
+          
+        when :params
+          params_hash = {}
+          request[:raw_hash].split(',').collect{|x| x.split('=>')}.each do |k,v|
+            key = k.gsub('"', '').gsub(' ', '').to_sym
+            value = v.gsub('"', '')
+            request.store(key, value)
+          end
+          
+          hash = block_given? ? yield(request) : request.hash
+
+          @actions[hash] ||= {:count => 0, :total_time => 0.0, :total_db_time => 0.0, :total_rendering_time => 0.0, 
+                                 :min_time => request[:duration], :max_time => request[:duration] }
+
+          @actions[hash][:count] += 1
+          request[:method] = 'GET' unless request[:method]
+          @methods[request[:method].upcase.to_sym] += 1
+
+          @hash_cache = hash
+        when :completed
+          @request_count += 1 
+          
+          @actions[@hash_cache][:total_time] += request[:dispatch_time]
+          @actions[@hash_cache][:mean_time] = @actions[@hash_cache][:total_time] / @actions[@hash_cache][:count].to_f          
+          @actions[@hash_cache][:min_time] = [@actions[@hash_cache][:min_time], request[:dispatch_time]].min
+          @actions[@hash_cache][:max_time] = [@actions[@hash_cache][:min_time], request[:dispatch_time]].max
+          
+          @actions[@hash_cache][:total_db_time] = 0
+          @actions[@hash_cache][:mean_db_time] = 0
+          @actions[@hash_cache][:mean_rendering_time] = 0
+          
+        when :failed
+
+      end
+    end
+  end
+end 

data/lib/rails_analyzer/log_parser.rb

@@ -1,83 +1,35 @@
-require 'date'
-
 module RailsAnalyzer
-  # Parse a rails log file
-  class LogParser 
 
+  class LogParser < Base::LogParser
+    
+    # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+    RAILS_21_COMPLETED = /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (?:\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(?:\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/
+
+    # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]  
+    RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
+    
+    
     LOG_LINES = {
       # Processing EmployeeController#index (for 123.123.123.123 at 2008-07-13 06:00:00) [GET]
       :started => {
         :teaser => /Processing/,
-        :regexp => /Processing (\w+)#(\w+) \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
-        :params => { :controller => 1, :action => 2, :ip => 3, :timestamp => 4, :method => 5}
+        :regexp => /Processing ((?:\w+::)?\w+)#(\w+)(?: to (\w+))? \(for (\d+\.\d+\.\d+\.\d+) at (\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d)\) \[([A-Z]+)\]/,
+        :params => [{:controller => :string}, {:action => :string}, {:format => :string}, {:ip => :string}, {:timestamp => :timestamp}, {:method => :string}]
       },
       # RuntimeError (Cannot destroy employee):  /app/models/employee.rb:198:in `before_destroy' 
       :failed => {
         :teaser => /Error/,
-        :regexp => /(\w+)(Error|Invalid) \((.*)\)\:(.*)/,
-        :params => { :error => 1, :exception_string => 3, :stack_trace => 4 }
+        :regexp => /(\w+)(?:Error|Invalid) \((.*)\)\:(.*)/,
+        :params => [{:error => :string}, {:exception_string => :string}, {:stack_trace => :string}]
       },
-      # Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://demo.nu/employees]
+    
       :completed => {
-        :teaser => /Completed/,
-        :regexp => /Completed in (\d+\.\d{5}) \(\d+ reqs\/sec\) (\| Rendering: (\d+\.\d{5}) \(\d+\%\) )?(\| DB: (\d+\.\d{5}) \(\d+\%\) )?\| (\d\d\d).+\[(http.+)\]/,
-        :params => { :url => 7, :status => [6, :to_i], :duration => [1, :to_f], :rendering => [3, :to_f], :db => [5, :to_f] }
+        :teaser   => /Completed in /,
+        :regexp   => Regexp.new("(?:#{RAILS_21_COMPLETED}|#{RAILS_22_COMPLETED})"),
+        :params   => [{:duration => :sec},  {:rendering => :sec},  {:db => :sec},  {:status => :int}, {:url => :string},  # 2.1 variant 
+                      {:duration => :msec}, {:rendering => :msec}, {:db => :msec}, {:status => :int}, {:url => :string}]  # 2.2 variant 
+
       }
     }
-    
-    # LogParser initializer
-    # <tt>file</tt> The fileobject this LogParser wil operate on.
-    def initialize(file, options = {})
-      @file_name = file
-      @options = options
-      @file_size = File.size(@file_name)
-    end
-    
-    def progress(&block)
-      @progress_handler = block
-    end
-
-    # Output a warning
-    # <tt>message</tt> The warning message (object)
-    def warn(message)
-      puts " -> " + message.to_s
-    end  
-
-    # Finds a log line and then parses the information in the line.
-    # Yields a hash containing the information found. 
-    # <tt>*line_types</tt> The log line types to look for (defaults to LOG_LINES.keys).
-    # Yeilds a Hash when it encounters a chunk of information.
-    def each(*line_types, &block)
-      # parse everything by default 
-      line_types = LOG_LINES.keys if line_types.empty?
-
-      File.open(@file_name) do |file|
-        
-        file.each_line do |line|
-          
-          @progress_handler.call(file.pos, @file_size) if @progress_handler
-          
-          line_types.each do |line_type|
-            if LOG_LINES[line_type][:teaser] =~ line
-              if LOG_LINES[line_type][:regexp] =~ line
-                request = { :type => line_type, :line => file.lineno }
-                LOG_LINES[line_type][:params].each do |key, value|
-                  request[key] = case value
-                    when Numeric; $~[value]
-                    when Array;   $~[value.first].send(value.last)
-                    else; nil
-                  end
-                  
-                end
-                yield(request) if block_given?
-              else
-                warn("Unparsable #{line_type} line: " + line[0..79]) unless line_type == :failed
-              end
-            end            
-          end
-        end
-        @progress_handler.call(:finished, @file_size) if @progress_handler
-      end      
-    end
   end
-end
+end

data/lib/rails_analyzer/record_inserter.rb

@@ -4,45 +4,8 @@ require 'sqlite3'
 module RailsAnalyzer
   
   # Set of functions that can be used to easily log requests into a SQLite3 Database.
-  class RecordInserter
+  class RecordInserter < Base::RecordInserter
     
-    attr_reader :database
-    attr_reader :current_request
-    attr_reader :warning_count
-    
-    # Initializer
-    # <tt>db_file</tt> The file which will be used for the SQLite3 Database storage.
-    def initialize(db_file)
-      @database = SQLite3::Database.new(db_file)
-      @insert_statements = nil
-      @warning_count = 0
-      create_tables_if_needed!
-    end
-        
-    # Calculate the database durations of the requests currenty in the database.
-    # Used if a logfile does contain any database durations.
-    def calculate_db_durations!
-      @database.execute('UPDATE "completed_queries" SET "database" = "duration" - "rendering" WHERE "database" IS NULL OR "database" = 0.0')
-    end
-    
-    # Insert a batch of loglines into the database.
-    # Function prepares insert statements, yeilds and then closes and commits.
-    def insert_batch(&block)
-      @database.transaction
-      prepare_statements!
-      block.call(self)
-      close_prepared_statements!
-      @database.commit
-    rescue Exception => e
-      puts e.message
-      @database.rollback
-    end
-    
-    def insert_warning(line, warning)
-      @database.execute("INSERT INTO parse_warnings (line, warning) VALUES (:line, :warning)", :line => line, :warning => warning)
-      @warning_count += 1
-    end
-        
     # Insert a request into the database.
     # <tt>request</tt> The request to insert.
     # <tt>close_statements</tt> Close prepared statements (default false)
@@ -72,90 +35,5 @@ module RailsAnalyzer
       close_prepared_statements! if close_statements
     end
     
-    # Insert a batch of files into the database.
-    # <tt>db_file</tt> The filename of the database file to use.
-    # Returns the created database.
-    def self.insert_batch_into(db_file, options = {}, &block)
-      db = RecordInserter.new(db_file)
-      db.insert_batch(&block)
-      return db
-    end    
-    
-    def count(type)
-      @database.get_first_value("SELECT COUNT(*) FROM \"#{type}_requests\"").to_i
-    end
-        
-    protected
-    
-    # Prepare insert statements.
-    def prepare_statements!
-      @insert_statements = {
-        :started => @database.prepare("
-            INSERT INTO started_requests ( line,  timestamp,  ip,  method,  controller,  action) 
-                                  VALUES (:line, :timestamp, :ip, :method, :controller, :action)"),
-                                  
-        :failed => @database.prepare("
-            INSERT INTO failed_requests ( line,  exception_string,  stack_trace,  error)
-                                 VALUES (:line, :exception_string, :stack_trace, :error)"),
-                                 
-        :completed => @database.prepare("
-            INSERT INTO completed_requests ( line,  url,  status,  duration,  rendering_time,  database_time)
-                                    VALUES (:line, :url, :status, :duration, :rendering, :db)")
-      }
-    end
-    
-    # Close all prepared statments
-    def close_prepared_statements!
-      @insert_statements.each { |key, stmt| stmt.close }
-    end
-
-    # Create the needed database tables if they don't exist.
-    def create_tables_if_needed!
-      
-      @database.execute("
-        CREATE TABLE IF NOT EXISTS started_requests (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          line INTEGER NOT NULL,
-          timestamp DATETIME NOT NULL, 
-          controller VARCHAR(255) NOT NULL, 
-          action VARCHAR(255) NOT NULL,
-          method VARCHAR(6) NOT NULL,          
-          ip VARCHAR(6) NOT NULL
-        )
-      ");
-
-      @database.execute("
-          CREATE TABLE IF NOT EXISTS failed_requests (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            line INTEGER NOT NULL,            
-            started_request_id INTEGER,        
-            error VARCHAR(255),    
-            exception_string VARCHAR(255),
-            stack_trace TEXT
-          )      
-      ");
-
-      @database.execute("
-        CREATE TABLE IF NOT EXISTS completed_requests (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          line INTEGER NOT NULL,          
-          started_request_id INTEGER,
-          url VARCHAR(255) NOT NULL,
-          hashed_url VARCHAR(255),
-          status INTEGER NOT NULL,
-          duration FLOAT,
-          rendering_time FLOAT,
-          database_time FLOAT
-        )
-      ");    
-      
-      @database.execute("CREATE TABLE IF NOT EXISTS parse_warnings (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          line INTEGER NOT NULL,       
-          warning VARCHAR(255) NOT NULL
-        )
-      ");
-    end
-
   end
 end

data/lib/rails_analyzer/summarizer.rb

@@ -1,37 +1,17 @@
+require 'date'
 module RailsAnalyzer
   
   # Functions to summarize an array of requets.
   # Can calculate request counts, duratations, mean times etc. of all the requests given.
-  class Summarizer
+  class Summarizer < Base::Summarizer
 
-    attr_reader :actions
-    attr_reader :errors
-    attr_reader :request_count
-    attr_reader :request_time_graph
-    attr_reader :first_request_at
-    attr_reader :last_request_at
-
-    attr_accessor :blocker_duration
-    DEFAULT_BLOCKER_DURATION = 1.0
-   
     # Initializer. Sets global variables
     # Options
     # * <tt>:calculate_database</tt> Calculate the database times if they are not explicitly logged.
-    def initialize(options = {})
-      @actions  = {}
-      @blockers = {}
-      @errors   = {}
-      @request_count = 0
-      @blocker_duration = DEFAULT_BLOCKER_DURATION
+    def initialize_hook(options = {})
       @calculate_database = options[:calculate_database]
-      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-    end
-   
-    # Check if any of the request parsed had a timestamp.
-    def has_timestamps?
-      @first_request_at
     end
-   
+
     # Parse a request string into a hash containing all keys found in the string.
     # Yields the hash found to the block operator.
     # <tt>request</tt> The request string to parse.
@@ -40,11 +20,18 @@ module RailsAnalyzer
       request[:duration] ||= 0
       
       case request[:type]   
-        when :started      
-          @first_request_at ||= request[:timestamp] # assume time-based order of file
-          @last_request_at  = request[:timestamp]   # assume time-based order of file
-          @request_time_graph[request[:timestamp][11..12].to_i] +=1
-
+        when :started 
+          if request[:timestamp]
+            @first_request_at ||= request[:timestamp] # assume time-based order of file
+            @last_request_at  = request[:timestamp]   # assume time-based order of file
+            @request_time_graph[request[:timestamp][11..12].to_i] +=1 
+          end
+          if request[:method]
+            @methods[request[:method].to_sym] ||= 0
+            @methods[request[:method].to_sym] += 1
+          else
+            @methods[:unknown] += 1
+          end
         when :completed
           @request_count += 1 
           hash = block_given? ? yield(request) : request.hash
@@ -82,40 +69,5 @@ module RailsAnalyzer
       end
     end
     
-    # Return a list of requests sorted on a specific action field
-    # <tt>field</tt> The action field to sort by.
-    # <tt>min_count</tt> Values which fall below this amount are not returned (default nil).
-    def sort_actions_by(field, min_count = nil)
-      actions = min_count.nil? ? @actions.to_a : @actions.delete_if { |k, v| v[:count] < min_count}.to_a
-      actions.sort { |a, b| (a[1][field.to_sym] <=> b[1][field.to_sym]) }
-    end
-
-    # Returns a list of request blockers sorted by a specific field
-    # <tt>field</tt> The action field to sort by.
-    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
-    def sort_blockers_by(field, min_count = @blocker_duration)
-      blockers = min_count.nil? ? @blockers.to_a : @blockers.delete_if { |k, v| v[:count] < min_count}.to_a
-      blockers.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
-    end
-
-    # Returns a list of request blockers sorted by a specific field
-    # <tt>field</tt> The action field to sort by.
-    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
-    def sort_errors_by(field, min_count = nil)
-      errors = min_count.nil? ? @errors.to_a : @errors.delete_if { |k, v| v[:count] < min_count}.to_a
-      errors.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
-    end
-
-    # Calculate the duration of a request
-    # Returns a DateTime object if possible, 0 otherwise.
-    def duration
-      (@last_request_at && @first_request_at) ? (DateTime.parse(@last_request_at) - DateTime.parse(@first_request_at)).round : 0
-    end
-    
-    # Check if the request time graph usable data.
-    def request_time_graph?
-      @request_time_graph.uniq != [0] && duration > 0
-    end
-
   end
 end 

data/output/hourly_spread.rb

@@ -7,7 +7,7 @@ if $summarizer.request_time_graph?
   color_cutoff      = 15
   
   puts
-  puts "Requests graph - per hour"
+  puts "Requests graph - average per day per hour"
   puts green("========================================================================")
   
   (0..23).each do |a|

data/output/timespan.rb

@@ -6,4 +6,11 @@ if $summarizer.has_timestamps?
   puts "Timestamp first request: #{$summarizer.first_request_at}"
   puts "Timestamp last request:  #{$summarizer.last_request_at}" 
   puts "Total time analyzed: #{$summarizer.duration} days"
-end
+end
+
+methods_print_array = []
+methods_request_count = $summarizer.methods.inject(0) { |subtotal, (k, v)| subtotal + v }
+$summarizer.methods.each do |key, value|
+  methods_print_array << "%s (%0.01f%%)" % [key, green((value * 100) / methods_request_count.to_f)]
+end
+puts 'Methods: ' + methods_print_array.join(', ') + '.'

data/output/usage.rb

@@ -7,6 +7,7 @@ puts "  --guess-database-time, -g:  Guesses the database duration of requests"
 puts "  --output, -o:               Comma-separated list of reports to show"     
 puts "  --amount, -c:               Displays the top <amount> elements in the reports"     
 puts "  --colorize, -z:             Fancy bash coloring"
+puts "  --merb, -m:                 Parse merb logfiles"
 puts 
 puts "Examples:"
 puts "  request-log-analyzer development.log"

data/test/log_fragments/fragment_3.log

@@ -0,0 +1,12 @@
+Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET]
+  Parameters: {"action"=>"demo", "controller"=>"page"}
+Logging in from session data...
+Logged in as willem@depillem.com
+Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain: 
+Rendering template within layouts/demo
+Rendering page/demo
+Rendered shared/_analytics (0.2ms)
+Rendered layouts/_actions (0.6ms)
+Rendered layouts/_menu (2.2ms)
+Rendered layouts/_tabbar (0.5ms)
+Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]

data/test/log_fragments/merb_1.log

@@ -0,0 +1,84 @@
+~ Loaded DEVELOPMENT Environment...
+~ Connecting to database...
+~ loading gem 'merb_datamapper' ...
+~ loading gem 'gettext' ...
+~ loading gem 'merb_helpers' ...
+~ loading gem 'merb-assets' ...
+~ loading gem 'merb-action-args' ...
+~ loading gem 'merb-mailer' ...
+~ loading gem 'merb_param_protection' ...
+~ loading gem 'merb_has_flash' ...
+~ loading gem 'merb_forgery_protection' ...
+~ loading gem 'dm-validations' ...
+~ loading gem 'dm-timestamps' ...
+~ loading gem 'dm-migrations' ...
+~ loading gem 'dm-aggregates' ...
+~ loading gem 'dm-adjust' ...
+~ loading gem 'dm-serializer' ...
+~ loading gem 'dm-constraints' ...
+~ loading gem 'dm-timeline' ...
+~ loading gem 'dm-searchable' ...
+~ loading gem 'dm-audited' ...
+~ loading gem 'lib/extensions' ...
+~ loading gem 'lib/authenticated_system/authenticated_dependencies' ...
+~ Compiling routes...
+~ Using 'share-nothing' cookie sessions (4kb limit per client)
+~ Using Mongrel adapter
+~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy", "method"=>"delete", "controller"=>"session"}
+~ Cookie deleted: auth_token => nil
+~ Redirecting to: / (302)
+~ {:dispatch_time=>0.243424, :after_filters_time=>6.9e-05, :before_filters_time=>0.213213, :action_time=>0.241652}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+~ Params: {"action"=>"index", "controller"=>"dashboard"}
+~ Redirecting to: /login (302)
+~ {:dispatch_time=>0.002649, :after_filters_time=>7.4e-05, :action_time=>0.001951}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:23 +0200 2008
+~ Params: {"action"=>"create", "controller"=>"session"}
+~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
+~ Params: {"authenticity_token"=>"[FILTERED]", "action"=>"create", "controller"=>"session", "login"=>"username", "password"=>"[FILTERED]", "remember_me"=>"0"}
+~ Redirecting to: / (302)
+~ {:dispatch_time=>0.006652, :after_filters_time=>0.000143, :before_filters_time=>0.000861, :action_time=>0.006171}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
+~ Params: {"action"=>"index", "controller"=>"dashboard"}
+~ Redirecting to: /dashboard (302)
+~ {:dispatch_time=>0.008241, :after_filters_time=>0.000126, :before_filters_time=>0.002632, :action_time=>0.007711}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:27 +0200 2008
+~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
+~ {:dispatch_time=>0.009458, :after_filters_time=>0.000103, :before_filters_time=>0.00266, :action_time=>0.008742}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:29 +0200 2008
+~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
+~ {:dispatch_time=>0.102725, :after_filters_time=>0.000115, :before_filters_time=>0.00411, :action_time=>0.101836}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:30 +0200 2008
+~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"organisations"}
+~ {:dispatch_time=>0.042575, :after_filters_time=>8.9e-05, :before_filters_time=>0.004267, :action_time=>0.041762}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:31 +0200 2008
+~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"}
+~ {:dispatch_time=>0.010311, :after_filters_time=>8.0e-05, :before_filters_time=>0.003195, :action_time=>0.009567}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:33 +0200 2008
+~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"}
+~ {:dispatch_time=>0.012913, :after_filters_time=>7.1e-05, :before_filters_time=>0.004422, :action_time=>0.012141}
+~ 
+
+~ Started request handling: Fri Aug 29 11:10:35 +0200 2008
+~ Params: {"action"=>"new", "namespace"=>"dashboard", "controller"=>"employees"}
+~ {:dispatch_time=>0.013051, :after_filters_time=>7.8e-05, :before_filters_time=>0.003576, :action_time=>0.011773}

data/test/merb_log_parser_test.rb

@@ -0,0 +1,39 @@
+require 'test/unit'
+
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/merb_analyzer/log_parser"
+
+class MerbLogParserTest < Test::Unit::TestCase
+  
+  def fragment_file(number)
+    "#{File.dirname(__FILE__)}/log_fragments/merb_#{number}.log"
+  end
+  
+  def test_parse_started_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:started) do |request|
+      requests << request
+    end
+    assert_equal requests[0][:timestamp], "Fri Aug 29 11:10:23 +0200 2008"
+  end
+
+  def test_parse_completed_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:completed) do |request|
+      requests << request
+    end
+    
+    assert_equal requests[0][:action_time], 0.241652
+  end
+
+  def test_parse_params_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:params) do |request|
+      requests << request
+    end
+    
+    assert_match '"controller"=>"session"', requests[0][:raw_hash]
+    assert_match '"action"=>"destroy"', requests[0][:raw_hash]
+  end
+  
+end

data/test/{log_parser_test.rb → rails_log_parser_test.rb}

@@ -1,17 +1,27 @@
 require 'test/unit'
 
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
 require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
 
-class LogParserTest < Test::Unit::TestCase
+class RailsLogParserTest < Test::Unit::TestCase
   
   def fragment_file(number)
     "#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
   end
   
   
+  def test_rails_22_log_format
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(3)).each(:completed) do |request|
+      count += 1
+      assert_equal 0.614, request[:duration]  # should be 0.614
+    end
+    assert_equal 1, count  
+  end
+  
   def test_progress_messages
     log_file = fragment_file(1)
-
+  
     finished_encountered = false
     file_size = File.size(log_file)
     
@@ -71,15 +81,15 @@ class LogParserTest < Test::Unit::TestCase
       assert_equal '10.1.1.33', request[:ip]
       assert_equal '2008-07-13 06:25:58', request[:timestamp]            
     end
-
+  
     parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) do |request|
       assert_equal "http://example.com/employee.xml", request[:url]
       assert_equal 200, request[:status]
       assert_equal 0.21665, request[:duration]
-      assert_equal 0.00926, request[:rendering]
+      assert_equal 0.00926, request[:rendering]      
       assert_equal 0.0, request[:db]
     end
-
+  
   end
   
 end

data/test/record_inserter_test.rb

@@ -1,5 +1,8 @@
 require 'test/unit'
 
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/base/record_inserter"
+
 require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
 require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/record_inserter"
 

data/test/tasks.rake

@@ -1,6 +1,6 @@
 require 'rake/testtask'
  
-desc 'Test the will_paginate plugin.'
+desc 'Unit test request-log-analyzer.'
 Rake::TestTask.new(:test) do |t|
   t.pattern = 'test/**/*_test.rb'
   t.verbose = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 0.1.2
+  version: 0.2.2
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-09-03 00:00:00 -07:00
+date: 2008-12-11 00:00:00 -08:00
 default_executable: request-log-analyzer
 dependencies: []
 
@@ -32,11 +32,18 @@ files:
 - bin/request-log-analyzer
 - bin/request-log-database
 - lib
+- lib/base
+- lib/base/log_parser.rb
+- lib/base/record_inserter.rb
+- lib/base/summarizer.rb
 - lib/bashcolorizer.rb
 - lib/command_line
 - lib/command_line/arguments.rb
 - lib/command_line/exceptions.rb
 - lib/command_line/flag.rb
+- lib/merb_analyzer
+- lib/merb_analyzer/log_parser.rb
+- lib/merb_analyzer/summarizer.rb
 - lib/rails_analyzer
 - lib/rails_analyzer/log_parser.rb
 - lib/rails_analyzer/record_inserter.rb
@@ -61,7 +68,10 @@ files:
 - test/log_fragments
 - test/log_fragments/fragment_1.log
 - test/log_fragments/fragment_2.log
-- test/log_parser_test.rb
+- test/log_fragments/fragment_3.log
+- test/log_fragments/merb_1.log
+- test/merb_log_parser_test.rb
+- test/rails_log_parser_test.rb
 - test/record_inserter_test.rb
 - test/tasks.rake
 has_rdoc: false
@@ -91,5 +101,6 @@ signing_key:
 specification_version: 2
 summary: A command line tool to analyze Rails logs
 test_files: 
-- test/log_parser_test.rb
+- test/merb_log_parser_test.rb
+- test/rails_log_parser_test.rb
 - test/record_inserter_test.rb