wvanbergen-request-log-analyzer 1.0.1 → 1.0.2

data/README.textile

@@ -1,49 +1,36 @@
-h1. Request log analyzer
+h1. Request-log-analyzer
 
 This is a simple command line tool to analyze request log files of both Rails and
-Merb. Its purpose is to find what actions are best candidates for optimization.
+Merb to produce a performance report. Its purpose is to find what actions are best candidates for optimization.
 
 * Analyzes Rails log files (all versions)
 * Can combine multiple files (handy if you are using logrotate)
 * Uses several metrics, including cumulative request time, average request time, process blockers, database and rendering time, HTTP methods and states, Rails action cache statistics, etc.) ("Sample output":http://wiki.github.com/wvanbergen/request-log-analyzer/sample-output)
 * Low memory footprint (server-safe)
-* MIT licensed
 * Fast
-
-h2. Additional information
-
-* "Project wiki at GitHub":http://wiki.github.com/wvanbergen/request-log-analyzer
-* "wvanbergen's blog posts":http://techblog.floorplanner.com/tag/request-log-analyzer/
+* MIT licensed
 
 h2. Installation
 
-@sudo gem install wvanbergen-request-log-analyzer --source http://gems.github.com@
+<pre>
+$ sudo gem install wvanbergen-request-log-analyzer --source http://gems.github.com
+</pre>
+
+To get the best results out of request-log-analyzer, make sure to 
+"set up logging correctly":http://wiki.github.com/wvanbergen/request-log-analyzer/configure-logging
+for your application.
 
 h2. Usage
 
+To analyze a log file and produce a performance report, run request-log-analyzer like this:
+
 <pre>
-  Usage: request-log-analyzer [LOGFILES*] <OPTIONS>
-
-  Input options:
-    --format <format>, -f:     Uses the specified log file format. Defaults to rails.
-    --after <date>             Only consider requests from <date> or later.
-    --before <date>            Only consider requests before <date>.
-    --select <field> <value>   Only consider requests where <field> matches <value>.
-    --reject <field> <value>   Only consider requests where <field> does not match <value>.
-
-  Output options:
-    --boring, -b               Output reports without ASCII colors.
-    --database <filename>, -d: Creates an SQLite3 database of all the parsed request information.
-    --debug                    Print debug information while parsing.
-    --file <filename>          Output to file.
-
-  Examples:
-    request-log-analyzer development.log
-    request-log-analyzer mongrel.0.log mongrel.1.log mongrel.2.log 
-    request-log-analyzer --format merb -d requests.db production.log
-
-  To install rake tasks in your Rails application, 
-  run the following command in your application's root directory:
-
-    request-log-analyzer install rails
+$ request-log-analyzer log/production.log
 </pre>
+
+For more details and available command line options, see the "project's wiki":http://wiki.github.com/wvanbergen/request-log-analyzer/basic-usage
+
+h2. Additional information
+
+* "Project wiki at GitHub":http://wiki.github.com/wvanbergen/request-log-analyzer
+* "wvanbergen's blog posts":http://techblog.floorplanner.com/tag/request-log-analyzer/

data/bin/request-log-analyzer

@@ -37,7 +37,6 @@ begin
     
     command_line.option(:format, :alias => :f, :default => 'rails')
     command_line.option(:file, :alias => :e)
-    command_line.switch(:single_lines, :s)
     command_line.switch(:assume_correct_order)
     
     command_line.option(:aggregator, :alias => :a, :multiple => true)

data/lib/request_log_analyzer/aggregator/base.rb

@@ -2,12 +2,6 @@ module RequestLogAnalyzer::Aggregator
 
   # The base class of an aggregator. This class provides the interface to which
   # every aggregator should comply (by simply subclassing this class).
-  #
-  # When building an aggregator, do not forget that RequestLogAnalyzer can run in
-  # single line mode or in combined requests mode. Make sure your aggregator can 
-  # handle both cases, or raise an exception if RLA is rnning in the wrong mode.
-  # Calling options[:combined_requests] tells you if RLA is running in combined 
-  # requests mode, otherwise it is running in single line mode.
   class Base
     
     include RequestLogAnalyzer::FileFormat::Awareness

data/lib/request_log_analyzer/aggregator/database.rb

@@ -5,43 +5,45 @@ module RequestLogAnalyzer::Aggregator
 
   class Database < Base
 
-    attr_reader :request_id
-
+    # Establishes a connection to the database and creates the necessary database schema for the
+    # current file format
     def prepare
       ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
-
-      File.unlink(options[:database]) if File.exist?(options[:database])
+      File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
       create_database_schema!
-      
-      @request_id = 0
     end
     
+    # Aggregates a request into the database
+    # This will create a record in the requests table and create a record for every line that has been parsed,
+    # in which the captured values will be stored.
     def aggregate(request)
-      @request_id += 1
-      
+      @request_object = @request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
       request.lines.each do |line|
-        class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
-        
         attributes = line.reject { |k, v| [:line_type].include?(k) }
-        attributes[:request_id] = @request_id if options[:combined_requests]  
-        file_format.class.const_get(class_name).create!(attributes)
+        @request_object.send("#{line[:line_type]}_lines").build(attributes)
       end
+      @request_object.save!
     rescue SQLite3::SQLException => e
       raise Interrupt, e.message
     end
     
+    # Finalizes the aggregator by closing the connection to the database
     def finalize
+      @request_count = @orm_module::Request.count
       ActiveRecord::Base.remove_connection
     end
     
+    # Records w warining in the warnings table.
     def warning(type, message, lineno)
-      file_format.class::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+      @orm_module::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
     end
     
+    # Prints a short report of what has been inserted into the database
     def report(output = STDOUT, report_width = 80, color = false)
       output << "\n"
       output << green("━" * report_width, color) + "\n"
       output <<  "A database file has been created with all parsed request information.\n"
+      output <<  "#{@request_count} requests have been added to the database.\n"      
       output <<  "To execute queries on this database, run the following command:\n"
       output <<  "  $ sqlite3 #{options[:database]}\n"
       output << "\n"
@@ -49,17 +51,47 @@ module RequestLogAnalyzer::Aggregator
     
     protected 
     
+    # This function creates a database table for a given line definition.
+    # It will create a field for every capture in the line, and adds a lineno field to indicate at
+    # what line in the original file the line was found, and a request_id to link lines related
+    # to the same request.
     def create_database_table(name, definition)
       ActiveRecord::Migration.verbose = options[:debug]
       ActiveRecord::Migration.create_table("#{name}_lines") do |t|
-        t.column(:request_id, :integer) #if options[:combined_requests]
+        t.column(:request_id, :integer)
         t.column(:lineno, :integer)
         definition.captures.each do |capture|
           t.column(capture[:name], column_type(capture))
         end
       end
     end
+    
+    # Creates an ActiveRecord class for a given line definition.
+    # A subclass of ActiveRecord::Base is created and an association with the Request class is
+    # created using belongs_to / has_many. This association will later be used to create records
+    # in the corresponding table. This table should already be created before this method is called.
+    def create_activerecord_class(name, definition)
+      class_name = "#{name}_line".camelize
+      klass = Class.new(ActiveRecord::Base)
+      klass.send(:belongs_to, :request)
+      @orm_module.const_set(class_name, klass) unless @orm_module.const_defined?(class_name)
+      @request_class.send(:has_many, "#{name}_lines".to_sym)
+    end    
+    
+    # Creates a requests table, in which a record is created for every request. It also creates an
+    # ActiveRecord::Base class to communicate with this table.
+    def create_request_table_and_class
+      ActiveRecord::Migration.verbose = options[:debug]
+      ActiveRecord::Migration.create_table("requests") do |t|
+        t.integer :first_lineno
+        t.integer :last_lineno
+      end    
+      
+      @orm_module.const_set('Request', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Request')     
+      @request_class = @orm_module.const_get('Request')
+    end
 
+    # Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
     def create_warning_table_and_class
       ActiveRecord::Migration.verbose = options[:debug]
       ActiveRecord::Migration.create_table("warnings") do |t|
@@ -68,23 +100,30 @@ module RequestLogAnalyzer::Aggregator
         t.integer :lineno          
       end    
       
-      file_format.class.const_set('Warning', Class.new(ActiveRecord::Base)) unless file_format.class.const_defined?('Warning')
+      @orm_module.const_set('Warning', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Warning')
     end
-
-    def create_activerecord_class(name, definition)
-      class_name = "#{name}_line".camelize
-      file_format.class.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.class.const_defined?(class_name)
-    end
-
+    
+    # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 
+    # file format definition. These ORM classes will later be used to create records in the database.
     def create_database_schema!
+      
+      if file_format.class.const_defined?('Database')
+        @orm_module = file_format.class.const_get('Database')
+      else
+        @orm_module = file_format.class.const_set('Database', Module.new)
+      end
+
+      create_request_table_and_class
+      create_warning_table_and_class
+      
       file_format.line_definitions.each do |name, definition|
         create_database_table(name, definition)
         create_activerecord_class(name, definition)
       end
-      
-      create_warning_table_and_class
     end
     
+    # Function to determine the column type for a field
+    # TODO: make more robust / include in file-format definition
     def column_type(capture)
       case capture[:type]
       when :sec;   :double

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -69,7 +69,7 @@ module RequestLogAnalyzer::Aggregator
        
     def report(output=STDOUT, report_width = 80, color = false)
       report_header(output, report_width, color)
-      if source.parsed_requests - source.skipped_requests > 0
+      if source.parsed_requests > 0
         @trackers.each { |tracker| tracker.report(output, report_width, color) }
       else
         output << "\n"
@@ -82,8 +82,8 @@ module RequestLogAnalyzer::Aggregator
       output << "Request summary\n"
       output << green("━" * report_width, color) + "\n"
       output << "Parsed lines:         #{green(source.parsed_lines, color)}\n"
-      output << "Parsed requests:      #{green(source.parsed_requests, color)}\n"  if options[:combined_requests]
-      output << "Skipped requests:     #{green(source.skipped_requests, color)}\n" if source.skipped_requests > 0
+      output << "Parsed requests:      #{green(source.parsed_requests, color)}\n"
+      output << "Skipped lines:        #{green(source.skipped_lines, color)}\n" if source.skipped_lines > 0
       if has_warnings?
         output <<  "Warnings:             " + @warnings_encountered.map { |(key, value)| "#{key.inspect}: #{blue(value, color)}" }.join(', ') + "\n"
       end

data/lib/request_log_analyzer/controller.rb

@@ -33,7 +33,6 @@ module RequestLogAnalyzer
             
       options = { :report_width => arguments[:report_width].to_i, :output => STDOUT}
 
-      options[:combined_requests] = !arguments[:single_lines]
       options[:database] = arguments[:database] if arguments[:database]
       options[:debug]    = arguments[:debug]
       options[:colorize] = !arguments[:boring]
@@ -45,7 +44,7 @@ module RequestLogAnalyzer
                 
       # Create the controller with the correct file format
       file_format = RequestLogAnalyzer::FileFormat.load(arguments[:format])
-      source_files = nil
+
       # register sources
       if arguments.parameters.length == 1
         file = arguments.parameters[0]
@@ -63,27 +62,22 @@ module RequestLogAnalyzer
       
       controller = Controller.new(RequestLogAnalyzer::Source::LogFile.new(file_format, options), options)
 
-      # register filters
-      # filters are only supported in combined requests mode
-      if options[:combined_requests]
-        
-        options[:assume_correct_order] = arguments[:assume_correct_order]
-        
-        if arguments[:after] || arguments[:before]
-          filter_options = {}
-          filter_options[:after]  = DateTime.parse(arguments[:after])  
-          filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
-          controller.add_filter(:timespan, filter_options)
-        end
-        
-        arguments[:reject].each do |(field, value)|
-          controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
-        end
-        
-        arguments[:select].each do |(field, value)|
-          controller.add_filter(:field, :mode => :select, :field => field, :value => value)
-        end
-        
+      options[:assume_correct_order] = arguments[:assume_correct_order]
+      
+      # register filters        
+      if arguments[:after] || arguments[:before]
+        filter_options = {}
+        filter_options[:after]  = DateTime.parse(arguments[:after])  
+        filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
+        controller.add_filter(:timespan, filter_options)
+      end
+      
+      arguments[:reject].each do |(field, value)|
+        controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
+      end
+      
+      arguments[:select].each do |(field, value)|
+        controller.add_filter(:field, :mode => :select, :field => field, :value => value)
       end
 
       # register aggregators
@@ -95,7 +89,9 @@ module RequestLogAnalyzer
     
       # register the echo aggregator in debug mode
       controller.add_aggregator(:echo) if arguments[:debug]
-            
+      
+      file_format.setup_environment(controller)
+          
       return controller
     end
 
@@ -103,7 +99,6 @@ module RequestLogAnalyzer
     # <tt>format</tt> Logfile format. Defaults to :rails
     # Options are passd on to the LogParser.
     # * <tt>:aggregator</tt> Aggregator array.
-    # * <tt>:combined_requests</tt> Combine multiline requests into a single request.
     # * <tt>:database</tt> Database the controller should use.
     # * <tt>:echo</tt> Output debug information.
     # * <tt>:silent</tt> Do not output any warnings.
@@ -186,7 +181,7 @@ module RequestLogAnalyzer
       
       begin
         @source.requests do |request|
-          @filters.each { |filter| request = filter.filter(request) }
+          #@filters.each { |filter| request = filter.filter(request) }
           @aggregators.each { |agg| agg.aggregate(request) } if request
         end
       rescue Interrupt => e

data/lib/request_log_analyzer/file_format.rb

@@ -73,5 +73,9 @@ module RequestLogAnalyzer
     def valid?
       line_definitions.detect { |(name, ld)| ld.header } && line_definitions.detect { |(name, ld)| ld.footer }
     end
+    
+    def setup_environment(controller)
+      
+    end
   end
 end

data/lib/request_log_analyzer/file_format/rails.rb

@@ -22,11 +22,11 @@ class RequestLogAnalyzer::FileFormat::Rails < RequestLogAnalyzer::FileFormat
   line_definition :failed do |line|
     line.footer = true
     line.regexp = /((?:[A-Z]\w+\:\:)*[A-Z]\w+) \((.*)\)(?: on line #(\d+) of .+)?\:(.*)/
-    line.captures << { :name => :error,            :type => :string } \
-                  << { :name => :exception_string, :type => :string } \
-                  << { :name => :line,             :type => :integer } \
-                  << { :name => :file,             :type => :string } \
-                  << { :name => :stack_trace,      :type => :string, :anonymize => true }
+    line.captures << { :name => :error,       :type => :string } \
+                  << { :name => :message,     :type => :string } \
+                  << { :name => :line,        :type => :integer } \
+                  << { :name => :file,        :type => :string } \
+                  << { :name => :stack_trace, :type => :string, :anonymize => true }
   end
 
 
@@ -63,37 +63,8 @@ class RequestLogAnalyzer::FileFormat::Rails < RequestLogAnalyzer::FileFormat
 
 
   REQUEST_CATEGORIZER = Proc.new do |request|
-    if request.combined?
-    
-      if request =~ :failed
-        format = request[:format] || 'html'
-        "#{request[:error]} in #{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-      else
-        format = request[:format] || 'html'
-        "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-      end
-    
-    else
-      case request.line_type
-      when :processing   
-        format = request[:format] || 'html'
-        "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
-      
-      when :completed
-        url = request[:url].downcase.split(/^http[s]?:\/\/[A-z0-9\.-]+/).last.split('?').first # only the relevant URL part
-        url << '/' if url[-1] != '/'[0] && url.length > 1 # pad a trailing slash for consistency
-
-        url.gsub!(/\/\d+-\d+-\d+(\/|$)/, '/:date/') # Combine all (year-month-day) queries
-        url.gsub!(/\/\d+-\d+(\/|$)/, '/:month/') # Combine all date (year-month) queries
-        url.gsub!(/\/\d+[\w-]*/, '/:id') # replace identifiers in URLs request[:url] # TODO: improve me
-        url
-      
-      when :failed
-        request[:error]
-      else
-        raise "Cannot group this request: #{request.inspect}" 
-      end
-    end
+    format = request[:format] || 'html'
+    "#{request[:controller]}##{request[:action]}.#{format} [#{request[:method]}]"
   end
 
   report do |analyze|

data/lib/request_log_analyzer/log_parser.rb

@@ -1,18 +1,14 @@
 module RequestLogAnalyzer
   
   # The LogParser class reads log data from a given source and uses a file format definition
-  # to parse all relevent information about requests from the file.
+  # to parse all relevent information about requests from the file. A FileFormat module should 
+  # be provided that contains the definitions of the lines that occur in the log data.
   #
-  # A FileFormat module should be provided that contains the definitions of the lines that 
-  # occur in the log data. The log parser can run in two modes:
-  # - In single line mode, it will emit every detected line as a separate request
-  # - In combined requests mode, it will combine the different lines from the line defintions
-  #   into one request, that will then be emitted. 
-  #
-  # The combined requests mode gives better information, but can be problematic if the log 
-  # file is unordered. This can be the case if data is written to the log file simultaneously 
-  # by different mongrel processes. This problem is detected by the parser, but the requests
-  # that are mixed up cannot be parsed. It will emit warnings when this occurs.
+  # De order in which lines occur is used to combine lines to a single request. If these lines 
+  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
+  # the log file simultaneously by different mongrel processes. This problem is detected by the 
+  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
+  # occurs.
   class LogParser
     
     include RequestLogAnalyzer::FileFormat::Awareness
@@ -79,8 +75,6 @@ module RequestLogAnalyzer
       unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
       raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
       
-      puts "Parsing mode: " + (options[:combined_requests] ? 'combined requests' : 'single lines') if options[:debug]
-      
       @current_io = io
       @current_io.each_line do |line|
         
@@ -94,11 +88,7 @@ module RequestLogAnalyzer
 
         if request_data
           @parsed_lines += 1
-          if @options[:combined_requests]
-            update_current_request(request_data, &block)
-          else
-            handle_request(RequestLogAnalyzer::Request.create(@file_format, request_data), &block)
-          end       
+          update_current_request(request_data, &block)
         end
       end
       
@@ -126,9 +116,9 @@ module RequestLogAnalyzer
     
     protected
     
-    # Combines the different lines of a request into a single Request object.
-    # This function is only called in combined requests mode. It will start a new request when
-    # a header line is encountered en will emit the request when a footer line is encountered.
+    # Combines the different lines of a request into a single Request object. It will start a 
+    # new request when a header line is encountered en will emit the request when a footer line 
+    # is encountered.
     #
     # - Every line that is parsed before a header line is ignored as it cannot be included in 
     #   any request. It will emit a :no_current_request warning.

data/lib/request_log_analyzer/request.rb

@@ -4,12 +4,6 @@ module RequestLogAnalyzer
   # Instances are created by the LogParser and are passed to the different aggregators, so they
   # can do their aggregating work. 
   #
-  # Note that RequestLogAnalyzer can run in two modes:
-  # - Single line mode: every parsed line is regarded as a request. Request::single_line? will 
-  #   return true in this case
-  # - Combined requests mode: lines that belong together are grouped into one request.
-  #   Request#combined? will return true in this case.
-  #
   # This class provides several methods to access the data that was parsed from the log files.
   # Request#first(field_name) returns the first (only) value corresponding to the given field
   # Request#every(field_name) returns all values corresponding to the given field name as array.
@@ -54,7 +48,7 @@ module RequestLogAnalyzer
     
     # Returns the value that was captured for the "field" of this request.
     # This function will return the first value that was captured if the field
-    # was captured in multiple lines for a combined request.
+    # was captured in multiple lines
     def first(field)
       @attributes[field]
     end
@@ -72,44 +66,30 @@ module RequestLogAnalyzer
       @lines.length == 0
     end
     
-    # Checks whether this request contains exactly one line. This means that RequestLogAnalyzer
-    # is running in single_line mode.
-    def single_line?
-      @lines.length == 1
-    end
-    
-    # Checks whether this request contains more than one line. This means that RequestLogAnalyzer
-    # is runring in combined requests mode.
-    def combined?
-      @lines.length > 1
-    end
-    
     # Checks whether this request is completed. A completed request contains both a parsed header
     # line and a parsed footer line. Not that calling this function in single line mode will always 
     # return false.
     def completed?
-      puts attributes[:method]
-
       header_found, footer_found = false, false
       @lines.each do |line| 
         line_def = file_format.line_definitions[line[:line_type]]
         header_found = true if line_def.header
         footer_found = true if line_def.footer        
       end
-      header_found && footer_found
-      
-    end
-    
-    # Returns the line type of the parsed line of this request.
-    # This function can only be called in single line mode.
-    def line_type
-      raise "Not a single line request!" unless single_line?
-      lines.first[:line_type]
+      header_found && footer_found      
     end
     
     # Returns the first timestamp encountered in a request. 
     def timestamp
       first(:timestamp)
     end
+    
+    def first_lineno
+      @lines.first[:lineno]
+    end
+    
+    def last_lineno
+      @lines.last[:lineno]
+    end
   end
 end

data/lib/request_log_analyzer/source/base.rb

@@ -15,8 +15,8 @@ module RequestLogAnalyzer::Source
     # The total number of parsed requests.
     attr_reader :parsed_requests
 
-    # The number of skipped requests because of date constraints
-    attr_reader :skipped_requests
+    # The number of skipped lines because of warnings
+    attr_reader :skipped_lines
 
     # Base source class used to filter input requests.
 

data/lib/request_log_analyzer/source/log_file.rb

@@ -1,17 +1,14 @@
 module RequestLogAnalyzer::Source
+  
   # The LogParser class reads log data from a given source and uses a file format definition
-  # to parse all relevent information about requests from the file.
+  # to parse all relevent information about requests from the file. A FileFormat module should 
+  # be provided that contains the definitions of the lines that occur in the log data.
   #
-  # A FileFormat module should be provided that contains the definitions of the lines that 
-  # occur in the log data. The log parser can run in two modes:
-  # - In single line mode, it will emit every detected line as a separate request
-  # - In combined requests mode, it will combine the different lines from the line defintions
-  #   into one request, that will then be emitted. 
-  #
-  # The combined requests mode gives better information, but can be problematic if the log 
-  # file is unordered. This can be the case if data is written to the log file simultaneously 
-  # by different mongrel processes. This problem is detected by the parser, but the requests
-  # that are mixed up cannot be parsed. It will emit warnings when this occurs.
+  # De order in which lines occur is used to combine lines to a single request. If these lines 
+  # are mixed, requests cannot be combined properly. This can be the case if data is written to 
+  # the log file simultaneously by different mongrel processes. This problem is detected by the 
+  # parser, but the requests that are mixed up cannot be parsed. It will emit warnings when this 
+  # occurs.
   class LogFile < RequestLogAnalyzer::Source::Base
 
     attr_reader :source_files
@@ -24,8 +21,8 @@ module RequestLogAnalyzer::Source
       @options          = options
       @parsed_lines     = 0
       @parsed_requests  = 0
-      @skipped_requests = 0
-      @current_io = nil
+      @skipped_lines    = 0
+      @current_io       = nil
       @source_files     = options[:source_files]
 
       # install the file format module (see RequestLogAnalyzer::FileFormat)
@@ -78,8 +75,6 @@ module RequestLogAnalyzer::Source
       unknown = line_types.reject { |line_type| file_format.line_definitions.has_key?(line_type) }
       raise "Unknown line types: #{unknown.join(', ')}" unless unknown.empty?
 
-      puts "Parsing mode: " + (options[:combined_requests] ? 'combined requests' : 'single lines') if options[:debug]
-
       @current_io = io
       @current_io.each_line do |line|
 
@@ -93,12 +88,7 @@ module RequestLogAnalyzer::Source
 
         if request_data
           @parsed_lines += 1
-          if @options[:combined_requests]
-            update_current_request(request_data, &block)
-          else
-            @parsed_requests +=1
-            yield RequestLogAnalyzer::Request.create(@file_format, request_data)
-          end       
+          update_current_request(request_data, &block)
         end
       end
 
@@ -126,9 +116,9 @@ module RequestLogAnalyzer::Source
 
     protected
 
-    # Combines the different lines of a request into a single Request object.
-    # This function is only called in combined requests mode. It will start a new request when
-    # a header line is encountered en will emit the request when a footer line is encountered.
+    # Combines the different lines of a request into a single Request object. It will start a 
+    # new request when a header line is encountered en will emit the request when a footer line 
+    # is encountered.
     #
     # - Every line that is parsed before a header line is ignored as it cannot be included in 
     #   any request. It will emit a :no_current_request warning.
@@ -144,7 +134,7 @@ module RequestLogAnalyzer::Source
             yield @current_request
             @current_request = RequestLogAnalyzer::Request.create(@file_format, request_data)
           else
-            @skipped_requests += 1
+            @skipped_lines += 1
             warn(:unclosed_request, "Encountered header line, but previous request was not closed!")
             @current_request = nil # remove all data that was parsed, skip next request as well.
           end
@@ -160,7 +150,7 @@ module RequestLogAnalyzer::Source
             @current_request = nil 
           end
         else
-          @skipped_requests += 1
+          @skipped_lines += 1
           warn(:no_current_request, "Parsebale line found outside of a request!")
         end
       end

data/spec/database_inserter_spec.rb

@@ -18,16 +18,31 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
   
   it "should create the correct tables" do
     ActiveRecord::Migration.should_receive(:create_table).with("warnings")    
+    ActiveRecord::Migration.should_receive(:create_table).with("requests")        
     ActiveRecord::Migration.should_receive(:create_table).with("first_lines")
     ActiveRecord::Migration.should_receive(:create_table).with("test_lines")        
     ActiveRecord::Migration.should_receive(:create_table).with("last_lines")    
     @database_inserter.prepare
   end
   
+  it "should create a default Request class" do
+    @database_inserter.prepare
+    SpecFormat::Database::Request.ancestors.should include(ActiveRecord::Base)
+    SpecFormat::Database::Request.column_names.should include('first_lineno')
+    SpecFormat::Database::Request.column_names.should include('last_lineno')    
+  end
+  
+  it "should create associations for the default Request class" do
+    @database_inserter.prepare
+    @request = SpecFormat::Database::Request.new
+    @request.should respond_to(:test_lines)
+    @request.test_lines.should
+  end
+  
   it "should create the default table names" do
     @database_inserter.prepare
     @database_inserter.file_format.line_definitions.each do |name, definition|
-      klass = SpecFormat.const_get("#{name}_line".camelize)
+      klass = SpecFormat::Database.const_get("#{name}_line".camelize)
       klass.column_names.should include('id')
       klass.column_names.should include('lineno')      
       klass.column_names.should include('request_id')      
@@ -37,9 +52,9 @@ describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
   it "should create the correct fields in the table" do
     @database_inserter.prepare
     
-    SpecFormat::FirstLine.column_names.should include('request_no')
-    SpecFormat::LastLine.column_names.should include('request_no')
-    SpecFormat::TestLine.column_names.should include('test_capture')    
+    SpecFormat::Database::FirstLine.column_names.should include('request_no')
+    SpecFormat::Database::LastLine.column_names.should include('request_no')
+    SpecFormat::Database::TestLine.column_names.should include('test_capture')    
   end
   
 end
@@ -52,8 +67,8 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
     @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
     @database_inserter.prepare
         
-    @single = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
-    @combined = RequestLogAnalyzer::Request.create(spec_format, 
+    @incomplete_request = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
+    @completed_request = RequestLogAnalyzer::Request.create(spec_format, 
                           {:line_type => :first, :request_no  => 564},
                           {:line_type => :test, :test_capture => "awesome"},
                           {:line_type => :test, :test_capture => "indeed"},                                                    
@@ -64,20 +79,22 @@ describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
     File.unlink(TEST_DATABASE_FILE) if File.exist?(TEST_DATABASE_FILE)
   end 
   
-  it "should insert a record in the relevant table" do
-    SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564))
-    @database_inserter.aggregate(@single)
+  it "should insert a record in the request table" do
+    SpecFormat::Database::Request.count.should == 0
+    @database_inserter.aggregate(@incomplete_request)
+    SpecFormat::Database::Request.count.should == 1
   end
   
-  it "should insert records in all relevant tables" do
-    SpecFormat::FirstLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
-    SpecFormat::TestLine.should_receive(:create!).twice
-    SpecFormat::LastLine.should_receive(:create!).with(hash_including(:request_no => 564)).once
-    @database_inserter.aggregate(@combined)
+  it "should insert records in all relevant line tables" do
+    @database_inserter.aggregate(@completed_request)
+    request = SpecFormat::Database::Request.first
+    request.should have(2).test_lines
+    request.should have(1).first_lines  
+    request.should have(1).last_lines    
   end
   
   it "should log a warning in the warnings table" do
-    SpecFormat::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
+    SpecFormat::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
     @database_inserter.warning(:test_warning, "Testing the warning system", 12)
   end
   

data/spec/fixtures/test_file_format.log

@@ -1,3 +1,4 @@
+processing request 1
 testing
 testing methods
 testing is cool
@@ -8,4 +9,5 @@ testing
 nonsense
 testing is
 
-more nonsense
+more nonsense
+finishing request 1

data/spec/log_parser_spec.rb

@@ -1,46 +1,12 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-describe RequestLogAnalyzer::LogParser, :single_line_requests do
-  
+describe RequestLogAnalyzer::LogParser, :requests do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
     @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
   end
   
-  it "should have line definitions" do
-    @log_parser.file_format.line_definitions.should_not be_empty
-  end
-  
-  # it "should have include the language specific hooks in the instance, not in the class" do
-  #   metaclass = (class << @log_parser; self; end)
-  #   metaclass.ancestors.should include(TestFileFormat::LogParser)
-  #   @log_parser.class.ancestors.should_not include(TestFileFormat::LogParser)
-  # end
-  
-  it "should parse a stream and find valid requests" do
-    io = File.new(log_fixture(:test_file_format), 'r')
-    @log_parser.parse_io(io, :line_types => [:test]) do |request| 
-      request.should be_kind_of(RequestLogAnalyzer::Request)
-      request.should =~ :test
-      request[:test_capture].should_not be_nil      
-    end
-    io.close
-  end
-  
-  it "should find as many lines as request" do
-    @log_parser.parse_file(log_fixture(:test_file_format)) { |request| request.should be_single_line }
-    @log_parser.parsed_lines.should eql(@log_parser.parsed_requests)
-  end
-end
-
-describe RequestLogAnalyzer::LogParser, :combined_requests do
-  include RequestLogAnalyzerSpecHelper
-  
-  before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
-  end
-  
   it "should have multiple line definitions" do
     @log_parser.file_format.line_definitions.length.should >= 2
   end  
@@ -51,7 +17,7 @@ describe RequestLogAnalyzer::LogParser, :combined_requests do
   
   it "should parse more lines than requests" do
     @log_parser.should_receive(:handle_request).with(an_instance_of(RequestLogAnalyzer::Request)).twice
-    @log_parser.parse_file(log_fixture(:test_language_combined)) { |request| request.should be_combined }  
+    @log_parser.parse_file(log_fixture(:test_language_combined))
     @log_parser.parsed_lines.should > 2    
   end
   
@@ -68,20 +34,32 @@ describe RequestLogAnalyzer::LogParser, :combined_requests do
       request[:test_capture].should == "amazing"      
     end
   end  
+  
+  it "should parse a stream and find valid requests" do
+    io = File.new(log_fixture(:test_file_format), 'r')
+    @log_parser.parse_io(io) do |request|
+      request.should be_kind_of(RequestLogAnalyzer::Request)
+      request.should =~ :test
+      request[:test_capture].should_not be_nil      
+    end
+    io.close
+  end  
+  
 end
 
 describe RequestLogAnalyzer::LogParser, :warnings do
   include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
+  end
     
   it "should warn about teaser matching problems" do
-    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
     @log_parser.should_receive(:warn).with(:teaser_check_failed, anything).exactly(5).times
     @log_parser.parse_file(log_fixture(:test_file_format))
   end
   
   it "should warn about unmatching request headers and footers" do
-    @log_parser = RequestLogAnalyzer::LogParser.new(spec_format, :combined_requests => true)
-    
     @log_parser.should_receive(:warn).with(:unclosed_request, anything).at_least(1).times    
     @log_parser.should_receive(:warn).with(:no_current_request, anything).at_least(1).times    
     @log_parser.should_not_receive(:handle_request)    

data/spec/merb_format_spec.rb

@@ -1,45 +1,25 @@
 # require File.dirname(__FILE__) + '/spec_helper'
 # 
-# describe RequestLogAnalyzer::LogParser, "Merb without combined requests" do
+# describe RequestLogAnalyzer::LogParser, "Merb" do
 #   include RequestLogAnalyzerSpecHelper
 #   
 #   before(:each) do
-#     @log_parser = RequestLogAnalyzer::LogParser.new(:merb, :combined_requests => false)
+#     @log_parser = RequestLogAnalyzer::LogParser.new(:merb)
+#   end
+#   
+#   it "should have a valid language definitions" do
+#     @log_parser.file_format.should be_valid
 #   end
 #   
 #   it "should parse a stream and find valid requests" do
 #     File.open(log_fixture(:merb), 'r') do |io|
 #       @log_parser.parse_io(io) do |request| 
 #         request.should be_kind_of(RequestLogAnalyzer::Request)
-#         request.should be_single_line
 #       end
 #     end
 #   end
-# 
-#   it "should find 33 request lines when lines are not linked" do
-#     @log_parser.should_receive(:handle_request).exactly(33).times
-#     @log_parser.parse_file(log_fixture(:merb))
-#   end  
-#   
-#   it "should find 11 request start lines when lines are not linked" do
-#     @log_parser.should_receive(:handle_request).exactly(11).times
-#     @log_parser.parse_file(log_fixture(:merb), :line_types => [:started])
-#   end  
-# end
-# 
-# 
-# describe RequestLogAnalyzer::LogParser, "Merb with combined requests" do
-#   include RequestLogAnalyzerSpecHelper
-#   
-#   before(:each) do
-#     @log_parser = RequestLogAnalyzer::LogParser.new(:merb, :combined_requests => true)
-#   end
-#   
-#   it "should have a valid language definitions" do
-#     @log_parser.file_format.should be_valid
-#   end
-#   
-#   it "should find 11 completed requests when lines are linked" do
+#
+#   it "should find 11 completed requests" do
 #     @log_parser.should_receive(:handle_request).exactly(11).times
 #     @log_parser.parse_file(log_fixture(:merb))
 #   end
@@ -55,4 +35,4 @@
 #     request[:before_filters_time].should == 0.213213
 #     request[:action_time].should == 0.241652
 #   end
-# end
+# end

data/spec/rails_format_spec.rb

@@ -1,10 +1,14 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-describe RequestLogAnalyzer::LogParser, "Rails without combined requests" do
+describe RequestLogAnalyzer::LogParser, "Rails" do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails), :combined_requests => false)
+    @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails))
+  end
+  
+  it "should have a valid language definitions" do
+    @log_parser.file_format.should be_valid
   end
   
   it "should parse a stream and find valid requests" do
@@ -13,30 +17,9 @@ describe RequestLogAnalyzer::LogParser, "Rails without combined requests" do
       request.should be_kind_of(RequestLogAnalyzer::Request)
     end
     io.close
-  end
-
-  it "should find 8 requests when lines are not linked" do
-    requests = []
-    @log_parser.parse_file(log_fixture(:rails_1x)) { |request| requests << request }
-    requests.length.should == 8
-    requests.each { |r| r.should be_single_line }
-    requests.select { |r| r.line_type == :processing }.should have(4).items
   end  
-end
-
-
-describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
-  include RequestLogAnalyzerSpecHelper
-  
-  before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails), :combined_requests => true)
-  end
-  
-  it "should have a valid language definitions" do
-    @log_parser.file_format.should be_valid
-  end
   
-  it "should find 4 completed requests when lines are linked" do
+  it "should find 4 completed requests" do
     @log_parser.should_not_receive(:warn)  
     @log_parser.should_receive(:handle_request).exactly(4).times
     @log_parser.parse_file(log_fixture(:rails_1x))
@@ -63,7 +46,6 @@ describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
     @log_parser.parse_file(log_fixture(:syslog_1x)) do |request| 
       
       request.should be_completed
-      request.should be_combined
       
       request[:controller].should == 'EmployeeController'
       request[:action].should     == 'index'
@@ -79,7 +61,6 @@ describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
     @log_parser.should_not_receive(:warn)
     @log_parser.parse_file(log_fixture(:rails_22_cached)) do |request| 
       request.should be_completed
-      request.should be_combined
       request =~ :cache_hit
     end  
   end

data/spec/request_spec.rb

@@ -1,80 +1,72 @@
 require File.dirname(__FILE__) + '/spec_helper'
 
-describe RequestLogAnalyzer::Request, :single_line do
+describe RequestLogAnalyzer::Request, :incomplete_request do
   
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @single_line_request = RequestLogAnalyzer::Request.new(spec_format)
-    @single_line_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
+    @incomplete_request = RequestLogAnalyzer::Request.new(spec_format)
+    @incomplete_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
   end
   
-  it "should include the file format module" do
-    (class << @single_line_request; self; end).ancestors.include?(TestFileFormat)
-  end  
-  
   it "should be single if only one line has been added" do
-    @single_line_request.should be_single_line
-    @single_line_request.should_not be_empty
-    @single_line_request.should_not be_combined    
+    @incomplete_request.should_not be_empty
   end
   
   it "should not be a completed request" do
-    @single_line_request.should_not be_completed
+    @incomplete_request.should_not be_completed
   end  
   
   it "should take the line type of the first line as global line_type" do
-    @single_line_request.line_type.should == :test
-    @single_line_request.should =~ :test
+    @incomplete_request.lines[0][:line_type].should == :test
+    @incomplete_request.should =~ :test
   end
   
   it "should return the first field value" do
-    @single_line_request[:test_capture].should == 'awesome!'
+    @incomplete_request[:test_capture].should == 'awesome!'
   end
   
   it "should return nil if no such field is present" do
-    @single_line_request[:nonexisting].should be_nil
+    @incomplete_request[:nonexisting].should be_nil
   end
 end  
 
 
-describe RequestLogAnalyzer::Request, :combined do
+describe RequestLogAnalyzer::Request, :completed_request do
   
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @combined_request = RequestLogAnalyzer::Request.new(spec_format)
-    @combined_request << { :line_type => :first, :lineno =>  1, :name => 'first line!' }    
-    @combined_request << { :line_type => :test,  :lineno =>  4, :test_capture => 'testing' }        
-    @combined_request << { :line_type => :test,  :lineno =>  7, :test_capture => 'testing some more' }            
-    @combined_request << { :line_type => :last,  :lineno => 10, :time => 0.03 }
+    @completed_request = RequestLogAnalyzer::Request.new(spec_format)
+    @completed_request << { :line_type => :first, :lineno =>  1, :name => 'first line!' }    
+    @completed_request << { :line_type => :test,  :lineno =>  4, :test_capture => 'testing' }        
+    @completed_request << { :line_type => :test,  :lineno =>  7, :test_capture => 'testing some more' }            
+    @completed_request << { :line_type => :last,  :lineno => 10, :time => 0.03 }
   end
   
-  it "should be a combined request when more lines are added" do
-    @combined_request.should be_combined
-    @combined_request.should_not be_single_line
-    @combined_request.should_not be_empty
+  it "should not be empty when multiple liness are added" do
+    @completed_request.should_not be_empty
   end
   
   it "should be a completed request" do
-    @combined_request.should be_completed
+    @completed_request.should be_completed
   end
   
   it "should recognize all line types" do
-    [:first, :test, :last].each { |type| @combined_request.should =~ type }
+    [:first, :test, :last].each { |type| @completed_request.should =~ type }
   end
   
   it "should detect the correct field value" do
-    @combined_request[:name].should == 'first line!'
-    @combined_request[:time].should == 0.03
+    @completed_request[:name].should == 'first line!'
+    @completed_request[:time].should == 0.03
   end
   
   it "should detect the first matching field value" do  
-    @combined_request.first(:test_capture).should == 'testing'
+    @completed_request.first(:test_capture).should == 'testing'
   end
   
   it "should detect the every matching field value" do  
-    @combined_request.every(:test_capture).should == ['testing', "testing some more"]
+    @completed_request.every(:test_capture).should == ['testing', "testing some more"]
   end
   
 end

data/spec/summarizer_spec.rb

@@ -1,13 +1,7 @@
 require File.dirname(__FILE__) + '/spec_helper'
 require 'request_log_analyzer/aggregator/summarizer'
 
-describe RequestLogAnalyzer::Aggregator::Summarizer, :single_line do
-  
-  before(:each) do
-  end
-end
-
-describe RequestLogAnalyzer::Aggregator::Summarizer, :combined_requests do
+describe RequestLogAnalyzer::Aggregator::Summarizer do
   
   before(:each) do
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-01-12 00:00:00 -08:00
+date: 2009-01-13 00:00:00 -08:00
 default_executable: request-log-analyzer
 dependencies: []