request-log-analyzer 1.2.9 → 1.3.0

data/lib/request_log_analyzer/database.rb

@@ -0,0 +1,177 @@
+require 'rubygems'
+require 'activerecord'
+
+class RequestLogAnalyzer::Database
+
+  def self.const_missing(const) # :nodoc:
+    RequestLogAnalyzer::load_default_class_file(self, const)
+  end  
+
+  include RequestLogAnalyzer::Database::Connection
+
+  attr_accessor :file_format
+  attr_reader :request_class, :warning_class, :source_class, :line_classes
+  
+  def initialize(connection_identifier = nil)
+    @line_classes = []
+    RequestLogAnalyzer::Database::Base.database = self
+    connect(connection_identifier)
+  end
+
+  # Returns the ORM class for the provided line type
+  def get_class(line_type)
+    line_type = line_type.name if line_type.respond_to?(:name)
+    Object.const_get("#{line_type}_line".camelize)
+  end
+  
+  # Returns the Request ORM class for the current database.
+  #
+  # It will create the class if not previously done so. The class will
+  # include a create_table! method the migrate the database.
+  def request_class
+    @request_class ||= begin
+      klass = Class.new(RequestLogAnalyzer::Database::Base) do
+        
+        def lines
+          @lines ||= begin
+            lines = []
+            self.class.reflections.each { |r, d| lines += self.send(r).all }
+            lines.sort
+          end
+        end
+        
+        # Creates the requests table
+        def self.create_table!
+          unless database.connection.table_exists?(:requests)
+            database.connection.create_table(:requests) do |t|
+              t.column :first_lineno, :integer
+              t.column :last_lineno,  :integer
+            end
+          end
+        end
+      end
+      
+      Object.const_set('Request', klass) 
+      Object.const_get('Request')
+    end
+  end
+  
+  # Returns the Source ORM class for the current database.
+  #
+  # It will create the class if not previously done so. The class will
+  # include a create_table! method the migrate the database.  
+  def source_class
+    @source_class ||= begin
+      klass = Class.new(RequestLogAnalyzer::Database::Base) do
+        
+        # Creates the sources table
+        def self.create_table!
+          unless database.connection.table_exists?(:sources)
+            database.connection.create_table(:sources) do |t|
+              t.column :filename, :string
+              t.column :mtime,    :datetime
+              t.column :filesize, :integer
+            end
+          end
+        end
+      end
+      
+      Object.const_set('Source', klass)
+      Object.const_get('Source')
+    end
+  end
+  
+  
+  # Returns the Warning ORM class for the current database.
+  #
+  # It will create the class if not previously done so. The class will
+  # include a create_table! method the migrate the database.  
+  def warning_class
+    @warning_class ||= begin
+      klass = Class.new(RequestLogAnalyzer::Database::Base) do
+        
+        # Creates the warnings table
+        def self.create_table!
+          unless database.connection.table_exists?(:warnings)
+            database.connection.create_table(:warnings) do |t|
+              t.column  :warning_type, :string, :limit => 30, :null => false
+              t.column  :message, :string
+              t.column  :source_id, :integer
+              t.column  :lineno, :integer
+            end
+          end
+        end
+      end
+      
+      Object.const_set('Warning', klass)
+      Object.const_get('Warning')
+    end
+  end
+  
+  # Loads the ORM classes by inspecting the tables in the current database
+  def load_database_schema!
+    connection.tables.map do |table|
+      case table.to_sym
+      when :warnings then warning_class
+      when :sources  then source_class
+      when :requests then request_class
+      else load_activerecord_class(table)
+      end
+    end
+  end
+  
+  # Returns an array of all the ActiveRecord-bases ORM classes for this database
+  def orm_classes
+    [warning_class, request_class, source_class] + line_classes
+  end
+  
+  # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be
+  # a table name or a LineDefinition instance.
+  def load_activerecord_class(linedefinition_or_table)
+    
+    case linedefinition_or_table
+    when String, Symbol
+      klass_name = linedefinition_or_table.to_s.singularize.camelize
+      klass      = RequestLogAnalyzer::Database::Base.subclass_from_table(linedefinition_or_table)
+    when RequestLogAnalyzer::LineDefinition
+      klass_name = "#{linedefinition_or_table.name}_line".camelize
+      klass      = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table)
+    end
+    
+    Object.const_set(klass_name, klass)
+    klass = Object.const_get(klass_name)
+    @line_classes << klass
+    return klass
+  end  
+
+  def fileformat_classes
+    raise "No file_format provided!" unless file_format
+    
+    default_classes = [request_class, source_class, warning_class]
+    line_classes    = file_format.line_definitions.map { |(name, definition)| load_activerecord_class(definition) }
+    return default_classes + line_classes
+  end
+
+  # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 
+  # file format definition. These ORM classes will later be used to create records in the database.
+  def create_database_schema!
+    fileformat_classes.each { |klass| klass.create_table! }
+  end
+  
+  # Drops the table of all the ORM classes, and unregisters the classes
+  def drop_database_schema!
+    file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!)
+    remove_orm_classes!
+  end
+  
+  # Unregisters every ORM class constant
+  def remove_orm_classes!
+    orm_classes.each do |klass|
+      if klass.respond_to?(:name) && !klass.name.blank?
+#        klass_base_name = klass.name.split('::').last
+        Object.send(:remove_const, klass.name) if Object.const_defined?(klass.name)
+      end
+    end
+  end
+
+end

data/lib/request_log_analyzer/file_format.rb

@@ -161,9 +161,12 @@ module RequestLogAnalyzer::FileFormat
 
     # Parses a line by trying to parse it using every line definition in this file format
     def parse_line(line, &warning_handler)
-      request_data = nil
-      self.line_definitions.any? { |lt, definition| request_data = definition.matches(line, &warning_handler) }
-      return request_data
+      self.line_definitions.each do |lt, definition|
+        match = definition.matches(line, &warning_handler)
+        return match if match
+      end
+      
+      return nil
     end
   end
 end

data/lib/request_log_analyzer/mailer.rb

@@ -0,0 +1,46 @@
+module RequestLogAnalyzer
+
+  class Mailer
+    
+    attr_accessor :data, :to, :host
+    
+    # Initialize a mailer
+    # <tt>to</tt> to address
+    # <tt>host</tt> the mailer host
+    # <tt>options</tt> Specific style options
+    def initialize(to, host = 'localhost', options = {})
+      require 'net/smtp' 
+      @to      = to
+      @host    = host
+      @options = options
+      @data    = []
+    end
+    
+    def mail
+      from        = @options[:from]        || 'contact@railsdoctors.com'
+      from_alias  = @options[:from_alias]  || 'Request-log-analyzer reporter'
+      to_alias    = @options[:to_alias]    || to
+      subject     = @options[:subjeect]    || "Request log analyzer report - generated on #{Time.now.to_s}"
+    msg = <<END_OF_MESSAGE
+From: #{from_alias} <#{from}>
+To: #{to_alias} <#{@to}>
+Subject: #{subject}
+
+#{@data.to_s}
+END_OF_MESSAGE
+      
+      Net::SMTP.start(@host) do |smtp|
+        smtp.send_message msg, from, to
+      end
+    end
+  
+    def << string
+      data << string
+    end
+    
+    def puts string
+      data << string
+    end
+
+  end
+end

data/lib/request_log_analyzer/request.rb

@@ -80,12 +80,13 @@ module RequestLogAnalyzer
       value_hash = parsed_line[:line_definition].convert_captured_values(parsed_line[:captures], self)
       value_hash[:line_type] = parsed_line[:line_definition].name
       value_hash[:lineno] = parsed_line[:lineno]
+      value_hash[:source] = parsed_line[:source]
       add_line_hash(value_hash)
     end
     
     def add_line_hash(value_hash)
       @lines << value_hash
-      @attributes = value_hash.merge(@attributes)      
+      @attributes = value_hash.merge(@attributes)
     end
     
     

data/lib/request_log_analyzer/source/{database.rb → database_loader.rb}

@@ -24,7 +24,7 @@ module RequestLogAnalyzer::Source
   end
 
   # The Database class gets log data from the database.
-  class Database < Base
+  class DatabaseLoader < Base
 
     attr_reader :source_files
     attr_reader :requests

data/lib/request_log_analyzer/source/log_parser.rb

@@ -11,6 +11,8 @@ module RequestLogAnalyzer::Source
   # that deal differently with this problem.
   class LogParser < Base
 
+    include Enumerable
+
     # The default parse strategy that will be used to parse the input.
     DEFAULT_PARSE_STRATEGY = 'assume-correct'
     
@@ -46,11 +48,11 @@ module RequestLogAnalyzer::Source
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
     # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
-    def each_request(options = {}, &block) # :yields: request
+    def each_request(options = {}, &block) # :yields: :request, request
       
       case @source_files
       when IO;     
-        puts "Parsing from the standard input. Press CTRL+C to finish."
+        puts "Parsing from the standard input. Press CTRL+C to finish." # FIXME: not here
         parse_stream(@source_files, options, &block) 
       when String
         parse_file(@source_files, options, &block) 
@@ -60,6 +62,9 @@ module RequestLogAnalyzer::Source
         raise "Unknown source provided"
       end
     end
+    
+    # Make sure the Enumerable methods work as expected
+    alias_method :each, :each_request
 
     # Parses a list of subsequent files of the same format, by calling parse_file for every
     # file in the array.
@@ -89,18 +94,22 @@ module RequestLogAnalyzer::Source
     # TODO: Fix progress bar that is broken for IO.popen, as it returns a single string.
     #
     # <tt>file</tt>:: The file that should be parsed.
-    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.    
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
     def parse_file(file, options = {}, &block)
 
-      @progress_handler.call(:started, file) if @progress_handler
+      @current_source = File.expand_path(file)
+      @progress_handler.call(:started, file)                  if @progress_handler
+      @source_changes_handler.call(:started, @current_source) if @source_changes_handler
       
       if decompress_file?(file).empty?
         File.open(file, 'r') { |f| parse_io(f, options, &block) }
       else
         IO.popen(decompress_file?(file), 'r') { |f| parse_io(f, options, &block) }
       end
-
-      @progress_handler.call(:finished, file) if @progress_handler
+      
+      @source_changes_handler.call(:finished, @current_source) if @source_changes_handler
+      @progress_handler.call(:finished, file)                  if @progress_handler
+      @current_source = nil
     end
 
     # Parses an IO stream. It will simply call parse_io. This function does not support progress updates
@@ -124,21 +133,19 @@ module RequestLogAnalyzer::Source
     # <tt>io</tt>:: The IO instance to use as source
     # <tt>options</tt>:: A hash of options that can be used by the parser.
     def parse_io(io, options = {}, &block) # :yields: request
-
-      @current_lineno = 0
+      @current_lineno = 1
       io.each_line do |line|
         @progress_handler.call(:progress, io.pos) if @progress_handler && io.kind_of?(File)
-
+        
         if request_data = file_format.parse_line(line) { |wt, message| warn(wt, message) }
           @parsed_lines += 1
-          update_current_request(request_data.merge(:lineno => @current_lineno), &block)
+          update_current_request(request_data.merge(:source => @current_source, :lineno => @current_lineno), &block)
         end
         
         @current_lineno += 1
       end
-
+      
       warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
-
       @current_lineno = nil
     end
 
@@ -149,10 +156,16 @@ module RequestLogAnalyzer::Source
     end
 
     # Add a block to this method to install a warning handler while parsing,
-    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages    
+    # <tt>proc</tt>:: The proc that will be called to handle parse warning messages
     def warning=(proc)
       @warning_handler = proc
     end
+    
+    # Add a block to this method to install a source change handler while parsing,
+    # <tt>proc</tt>:: The proc that will be called to handle source changes
+    def source_changes=(proc)
+      @source_changes_handler = proc
+    end
 
     # This method is called by the parser if it encounteres any parsing problems.
     # It will call the installed warning handler if any. 
@@ -229,11 +242,11 @@ module RequestLogAnalyzer::Source
     # - It will update the parsed_requests and skipped_requests variables accordingly
     #
     # <tt>request</tt>:: The parsed request instance (RequestLogAnalyzer::Request)
-    def handle_request(request, &block) # :yields: request
+    def handle_request(request, &block) # :yields: :request, request
       @parsed_requests += 1
       request.validate
       accepted = block_given? ? yield(request) : true
-      @skipped_requests += 1 if not accepted
+      @skipped_requests += 1 unless accepted
     end    
 
     # Checks whether a given line hash is a header line according to the current file format.

data/lib/request_log_analyzer/source.rb

@@ -33,11 +33,16 @@ module RequestLogAnalyzer::Source
     # The total number of parsed lines
     attr_reader :parsed_lines
 
+    # The number of skipped lines because of warnings
+    attr_reader :skipped_lines
+
     # The total number of parsed requests.
     attr_reader :parsed_requests
 
-    # The number of skipped lines because of warnings
-    attr_reader :skipped_lines
+    # The total number of skipped requests because of filters.
+    attr_reader :skipped_requests
+
+
 
     # Initializer, which will register the file format and save any options given as a hash.
     # <tt>format</tt>:: The file format instance

data/lib/request_log_analyzer.rb

@@ -11,23 +11,20 @@ module RequestLogAnalyzer
   
   # The current version of request-log-analyzer.
   # This will be diplayed in output reports etc.  
-  VERSION = "1.2.9"
-  
+  VERSION = "1.3.0"
+
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
   def self.const_missing(const)
     load_default_class_file(RequestLogAnalyzer, const)
   end
-    
+
   # Loads constants that reside in the RequestLogAnalyzer tree using the constant name
   # and its base constant to determine the filename.
   # <tt>base</tt>:: The base constant to load the constant from. This should be Foo when the constant Foo::Bar is being loaded.
   # <tt>const</tt>:: The constant to load from the base constant as a string or symbol. This should be 'Bar' or :Bar when the constant Foo::Bar is being loaded.
   def self.load_default_class_file(base, const)
-    path     = to_underscore(base.to_s)
-    basename = to_underscore(const.to_s)
-    filename = "#{File.dirname(__FILE__)}/#{path}/#{basename}"
-    require filename
+    require "#{to_underscore("#{base.name}::#{const}")}"
     base.const_get(const)
   end
 
@@ -36,7 +33,7 @@ module RequestLogAnalyzer
   # <tt>str</tt>:: The string to convert in the following format: <tt>ModuleName::ClassName</tt>
   def self.to_underscore(str)
     str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
-  end  
+  end
 
   # Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase 
   # (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.

data/request-log-analyzer.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
-  s.name    = 'request-log-analyzer'
-  s.version = "1.2.9"
-  s.date    = "2009-09-08"
+  s.name    = "request-log-analyzer"
+  s.version = "1.3.0"
+  s.date    = "2009-09-12"
   
   s.rubyforge_project = 'r-l-a'
   
@@ -9,14 +9,14 @@ Gem::Specification.new do |s|
   s.executables        = ['request-log-analyzer']
   s.default_executable = 'request-log-analyzer'
   
-  s.summary     = "A command line tool to analyze request logs for Rails, Merb and other application servers"
+  s.summary     = "A command line tool to analyze request logs for Apache, Rails, Merb and other application servers"
   s.description = <<-eos
     Request log analyzer's purpose is to find ot how your web application is being used and to focus your optimization efforts.
     This tool will parse all requests in the application's log file and aggregate the information. Once it is finished parsing 
     the log file(s), it will show the requests that take op most server time using various metrics. It can also insert all 
     parsed request information into a database so you can roll your own analysis. It supports Rails- and Merb-based applications 
-    out of the box, but file formats of other applications can easily be supported by supplying an easy to write log file format 
-    definition.
+    and Apache access log files out of the box, but file formats of other applications can easily be supported by supplying an 
+    easy to write log file format definition.
   eos
   
   s.rdoc_options << '--title' << s.name << '--main' << 'README.rdoc' << '--line-numbers' << '--inline-source'
@@ -31,6 +31,6 @@ Gem::Specification.new do |s|
   s.email    = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
   s.homepage = 'http://railsdoctors.com'
 
-  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/file_format/rails_development.rb spec/lib/macros.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/fixtures/decompression.log.bz2 lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb spec/fixtures/rails_unordered.log bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/matchers.rb spec/fixtures/test_order.log lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb spec/lib/testing_format.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb Rakefile spec/unit/filter/filter_spec.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/test_language_combined.log spec/fixtures/decompression.tar.gz spec/unit/filter/field_filter_spec.rb spec/spec.opts lib/request_log_analyzer/aggregator/database.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/lib/helpers.rb spec/fixtures/rails_1x.log spec/lib/mocks.rb spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/aggregator/database_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/source/database.rb tasks/github-gem.rake lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log spec/fixtures/syslog_1x.log LICENSE spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
-  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/aggregator/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
+  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log spec/lib/macros.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/database.rb spec/fixtures/decompression.log.bz2 lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb spec/fixtures/rails_unordered.log bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/testing_format.rb spec/fixtures/test_order.log spec/fixtures/rails.db lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb lib/request_log_analyzer/mailer.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb spec/database.yml Rakefile lib/request_log_analyzer/database/connection.rb spec/unit/filter/filter_spec.rb spec/fixtures/test_language_combined.log lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/decompression.tar.gz spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb lib/cli/database_console.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/lib/mocks.rb spec/lib/helpers.rb lib/cli/database_console_init.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/fixtures/rails_1x.log spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log tasks/github-gem.rake spec/unit/database/database_spec.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log spec/fixtures/syslog_1x.log LICENSE lib/request_log_analyzer/source/database_loader.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
+  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
 end

data/spec/database.yml

@@ -0,0 +1,17 @@
+sqlite3:
+  adapter: "sqlite3"
+  database: ":memory:"
+
+# mysql:
+#   adapter: "mysql"
+#   host: "localhost"
+#   username: "root"
+#   password:
+#   database: "rla_test"
+
+# postgresql:
+#   adapter: "postgresql"
+#   host: "localhost"
+#   username: "rla"
+#   password: "rla"
+#   database: "rla_test"

data/spec/integration/command_line_usage_spec.rb

@@ -12,17 +12,17 @@ describe RequestLogAnalyzer, 'running from command line' do
 
   it "should find 4 requests in default mode" do
     output = run("#{log_fixture(:rails_1x)}")
-    output.detect { |line| /Parsed requests\:\s*4/ =~ line }.should_not be_nil
+    output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
   end
 
-  it "should find 3 requests with a --select option" do
+  it "should skip 1 requests with a --select option" do
     output = run("#{log_fixture(:rails_1x)} --select controller PeopleController")
-    output.detect { |line| /Parsed requests\:\s*4/ =~ line }.should_not be_nil
+    output.any? { |line| /^Skipped requests\:\s*1\s/ =~ line }.should be_true
   end
 
-  it "should find 1 requests with a --reject option" do
+  it "should skip 3 requests with a --reject option" do
     output = run("#{log_fixture(:rails_1x)} --reject controller PeopleController")
-    output.detect { |line| /Parsed requests\:\s*4/ =~ line }.should_not be_nil
+    output.any? { |line| /^Skipped requests\:\s*3\s/ =~ line }.should be_true
   end
 
   it "should write output to a file with the --file option" do
@@ -37,7 +37,7 @@ describe RequestLogAnalyzer, 'running from command line' do
 
   it "should write HTML if --output HTML is provided" do
     output = run("#{log_fixture(:rails_1x)} --output HTML")
-    output.any? { |line| /<html.*>/ =~ line}
+    output.any? { |line| /<html[^>]*>/ =~ line}.should be_true
   end
 
   it "should run with the --database option" do
@@ -57,17 +57,22 @@ describe RequestLogAnalyzer, 'running from command line' do
 
   it "should parse a Merb file if --format merb is set" do
     output = run("#{log_fixture(:merb)} --format merb")
-    output.detect { |line| /Parsed requests\:\s*11/ =~ line }.should_not be_nil
+    output.any? { |line| /Parsed requests\:\s*11/ =~ line }.should be_true
   end
 
   it "should parse a Apache access log file if --apache-format is set" do
     output = run("#{log_fixture(:apache_combined)} --apache-format combined")
-    output.detect { |line| /Parsed requests\:\s*5/ =~ line }.should_not be_nil
+    output.any? { |line| /Parsed requests\:\s*5/ =~ line }.should be_true
   end
 
   it "should dump the results to a YAML file" do
     run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:dump)}")
     File.exist?(temp_output_file(:dump)).should be_true
     YAML::load(File.read(temp_output_file(:dump))).should have_at_least(1).item
-  end  
+  end
+  
+  it "should parse 4 requests from the standard input" do
+    output = run("- < #{log_fixture(:rails_1x)}")
+    output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
+  end
 end

data/spec/lib/macros.rb

@@ -1,2 +1,18 @@
 module RequestLogAnalyzer::Spec::Macros
+  
+  def test_databases
+    require 'yaml'
+    hash = YAML.load(File.read("#{File.dirname(__FILE__)}/../database.yml"))
+    hash.inject({}) { |res, (name, h)| res[name] = h.map { |(k,v)| "#{k}=#{v}" }.join(';'); res  }
+  end
+  
+  # Create or return a new TestingFormat
+  def testing_format
+    @testing_format ||= TestingFormat.create
+  end
+  
+  def default_orm_class_names
+    ['Warning', 'Request', 'Source']
+  end
+
 end

data/spec/lib/mocks.rb

@@ -9,6 +9,7 @@ module RequestLogAnalyzer::Spec::Mocks
     
     source.stub!(:warning=)
     source.stub!(:progress=)
+    source.stub!(:source_changes=)
 
     source.stub!(:prepare)
     source.stub!(:finalize)
@@ -43,15 +44,26 @@ module RequestLogAnalyzer::Spec::Mocks
     return output
   end
   
-  def mock_migrator
+  def mock_database(*stubs)
+    database = mock('RequestLogAnalyzer::Database')
+    database.stub!(:connect)
+    database.stub!(:disconnect)
+    database.stub!(:connection).and_return(mock_connection)
+    stubs.each { |s| database.stub!(s)}
+    return database
+  end
+  
+  def mock_connection
     table_creator = mock('ActiveRecord table creator')
     table_creator.stub!(:column)
     
-    migrator = mock('ActiveRecord::Base.connection for migrations')
-    migrator.stub!(:add_index)
-    migrator.stub!(:create_table).and_yield(table_creator).and_return(true)
-    migrator.stub!(:table_creator).and_return(table_creator)
-    return migrator
+    connection = mock('ActiveRecord::Base.connection')
+    connection.stub!(:add_index)
+    connection.stub!(:remove_index)
+    connection.stub!(:table_exists?).and_return(false)
+    connection.stub!(:create_table).and_yield(table_creator).and_return(true)
+    connection.stub!(:table_creator).and_return(table_creator)
+    return connection
   end