request-log-analyzer 1.2.1 → 1.2.6

data/.gitignore

@@ -0,0 +1,10 @@
+.svn/
+.DS_Store
+request-log-analyzer-*.gem
+requests.db
+/pkg
+/doc
+/tmp
+/classes
+/files
+/coverage

data/Rakefile

@@ -1,3 +1,5 @@
 Dir[File.dirname(__FILE__) + "/tasks/*.rake"].each { |file| load(file) }
+
+GithubGem::RakeTasks.new(:gem)
  
-task :default => :spec
+task :default => "spec:specdoc"

data/lib/request_log_analyzer/aggregator/database.rb

@@ -16,10 +16,13 @@ module RequestLogAnalyzer::Aggregator
   # created to log all parse warnings.
   class Database < Base
 
+    attr_reader :request_class, :request_count, :orm_module, :warning_class
+
     # Establishes a connection to the database and creates the necessary database schema for the
     # current file format
     def prepare
-      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
+      initialize_orm_module!
+      establish_database_connection!
       File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
       create_database_schema!
     end
@@ -28,9 +31,9 @@ module RequestLogAnalyzer::Aggregator
     # This will create a record in the requests table and create a record for every line that has been parsed,
     # in which the captured values will be stored.
     def aggregate(request)
-      @request_object = @request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      @request_object = request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
       request.lines.each do |line|
-        class_columns = @orm_module.const_get("#{line[:line_type]}_line".classify).column_names.reject { |column| ['id'].include?(column) }
+        class_columns = orm_module.const_get("#{line[:line_type]}_line".classify).column_names.reject { |column| ['id'].include?(column) }
         attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
         @request_object.send("#{line[:line_type]}_lines").build(attributes)
       end
@@ -41,13 +44,14 @@ module RequestLogAnalyzer::Aggregator
     
     # Finalizes the aggregator by closing the connection to the database
     def finalize
-      @request_count = @orm_module::Request.count
-      ActiveRecord::Base.remove_connection
+      @request_count = orm_module::Request.count
+      remove_database_connection!
+      deinitialize_orm_module!
     end
     
     # Records w warining in the warnings table.
     def warning(type, message, lineno)
-      @orm_module::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+      warning_class.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
     end
     
     # Prints a short report of what has been inserted into the database
@@ -61,91 +65,119 @@ module RequestLogAnalyzer::Aggregator
       output << "\n"
     end
     
+    # Retreives the connection that is used for the database inserter
+    def connection
+      orm_module::Base.connection
+    end    
+    
     protected 
     
+    # Create a module and a default subclass of ActiveRecord::Base on which to establish the database connection
+    def initialize_orm_module!
+      
+      # Create a Database module in the file format if it does not yet exists
+      file_format.class.const_set('Database', Module.new) unless file_format.class.const_defined?('Database')
+      @orm_module = file_format.class.const_get('Database')
+      
+      # Register the base activerecord class
+      unless orm_module.const_defined?('Base')
+        orm_base_class = Class.new(ActiveRecord::Base)
+        orm_base_class.abstract_class = true
+        orm_module.const_set('Base', orm_base_class)
+      end
+    end
+
+    # Deinitializes the ORM module and the ActiveRecord::Base subclass.
+    def deinitialize_orm_module!
+      file_format.class.send(:remove_const, 'Database') if file_format.class.const_defined?('Database')
+      @orm_module = nil
+    end
+
+    # Established a connection with the database for this session
+    def establish_database_connection!
+      orm_module::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
+      #ActiveRecord::Migration.class_eval("def self.connection; #{@orm_module.to_s}::Base.connection; end ")
+    end
+
+    def remove_database_connection!
+      #ActiveRecord::Migration.class_eval("def self.connection; ActiveRecord::Base.connection; end ")      
+      orm_module::Base.remove_connection
+    end
+    
     # This function creates a database table for a given line definition.
     # It will create a field for every capture in the line, and adds a lineno field to indicate at
     # what line in the original file the line was found, and a request_id to link lines related
     # to the same request. It will also create an index in the request_id field to speed up queries.
-    def create_database_table(name, definition)
-      ActiveRecord::Migration.verbose = options[:debug]
-      ActiveRecord::Migration.create_table("#{name}_lines") do |t|
+    def create_database_table(definition)
+      connection.create_table("#{definition.name}_lines") do |t|
+        
+        # Add default fields for every line type
         t.column(:request_id, :integer)
         t.column(:lineno, :integer)
+        
         definition.captures.each do |capture|
-
           # Add a field for every capture
           t.column(capture[:name], column_type(capture[:type]))
-          
-          # If the capture provides other field as well, create them too
-          if capture[:provides].kind_of?(Hash)
-            capture[:provides].each do |field, field_type|
-              t.column(field, column_type(field_type))
-            end
-          end
+
+          # If the capture provides other field as well, create columns for them, too
+          capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
         end
       end
-      ActiveRecord::Migration.add_index("#{name}_lines", [:request_id])
+      
+      # Create an index on the request_id column to support faster querying
+      connection.add_index("#{definition.name}_lines", [:request_id])
     end
     
     # Creates an ActiveRecord class for a given line definition.
     # A subclass of ActiveRecord::Base is created and an association with the Request class is
     # created using belongs_to / has_many. This association will later be used to create records
     # in the corresponding table. This table should already be created before this method is called.
-    def create_activerecord_class(name, definition)
-      class_name = "#{name}_line".camelize
-      klass = Class.new(ActiveRecord::Base)
+    def create_activerecord_class(definition)
+      class_name = "#{definition.name}_line".camelize
+      klass = Class.new(orm_module::Base)
       klass.send(:belongs_to, :request)
       
-      definition.captures.each do |capture|
-        klass.send(:serialize, capture[:name], Hash) if capture[:provides]
+      definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
+        klass.send(:serialize, capture[:name], Hash)
       end
       
-      @orm_module.const_set(class_name, klass) unless @orm_module.const_defined?(class_name)
-      @request_class.send(:has_many, "#{name}_lines".to_sym)
+      orm_module.const_set(class_name, klass) unless orm_module.const_defined?(class_name)
+      request_class.send(:has_many, "#{definition.name}_lines".to_sym)
     end    
     
     # Creates a requests table, in which a record is created for every request. It also creates an
     # ActiveRecord::Base class to communicate with this table.
     def create_request_table_and_class
-      ActiveRecord::Migration.verbose = options[:debug]
-      ActiveRecord::Migration.create_table("requests") do |t|
-        t.integer :first_lineno
-        t.integer :last_lineno
+      connection.create_table("requests") do |t|
+        t.column :first_lineno, :integer
+        t.column :last_lineno,  :integer
       end    
       
-      @orm_module.const_set('Request', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Request')     
-      @request_class = @orm_module.const_get('Request')
+      orm_module.const_set('Request', Class.new(orm_module::Base)) unless orm_module.const_defined?('Request')     
+      @request_class = orm_module.const_get('Request')
     end
 
     # Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
     def create_warning_table_and_class
-      ActiveRecord::Migration.verbose = options[:debug]
-      ActiveRecord::Migration.create_table("warnings") do |t|
-        t.string  :warning_type, :limit => 30, :null => false
-        t.string  :message
-        t.integer :lineno          
+      connection.create_table("warnings") do |t|
+        t.column  :warning_type, :string, :limit => 30, :null => false
+        t.column  :message, :string
+        t.column  :lineno, :integer
       end    
       
-      @orm_module.const_set('Warning', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Warning')
+      orm_module.const_set('Warning', Class.new(orm_module::Base)) unless orm_module.const_defined?('Warning')
+      @warning_class = orm_module.const_get('Warning')
     end
     
     # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 
     # file format definition. These ORM classes will later be used to create records in the database.
     def create_database_schema!
-      
-      if file_format.class.const_defined?('Database')
-        @orm_module = file_format.class.const_get('Database')
-      else
-        @orm_module = file_format.class.const_set('Database', Module.new)
-      end
-
       create_request_table_and_class
       create_warning_table_and_class
       
       file_format.line_definitions.each do |name, definition|
-        create_database_table(name, definition)
-        create_activerecord_class(name, definition)
+        create_database_table(definition)
+        create_activerecord_class(definition)
       end
     end
     
@@ -154,6 +186,7 @@ module RequestLogAnalyzer::Aggregator
     def column_type(type_indicator)
       case type_indicator
       when :eval;      :text
+      when :hash;      :text
       when :text;      :text
       when :string;    :string
       when :sec;       :double

data/lib/request_log_analyzer/controller.rb

@@ -191,14 +191,11 @@ module RequestLogAnalyzer
     def run!
       
       @aggregators.each { |agg| agg.prepare }
+      install_signal_handlers
       
-      begin
-        @source.each_request do |request|
-          aggregate_request(filter_request(request))
-        end
-      rescue Interrupt => e
-        handle_progress(:interrupted)
-        puts "Caught interrupt! Stopped parsing."
+      @source.each_request do |request|
+        aggregate_request(filter_request(request))
+        break if @interrupted
       end
 
       @aggregators.each { |agg| agg.finalize }
@@ -219,5 +216,13 @@ module RequestLogAnalyzer
       end
     end
     
+    def install_signal_handlers
+      Signal.trap("INT") do
+        handle_progress(:interrupted)
+        puts "Caught interrupt! Stopping parsing..."
+        @interrupted = true
+      end
+    end
+    
   end
 end

data/lib/request_log_analyzer/file_format/merb.rb

@@ -16,18 +16,16 @@ module RequestLogAnalyzer::FileFormat
       line.teaser = /Params/
       line.regexp = /Params\:\ (\{.+\})/
       line.captures << { :name => :params, :type => :eval, :provides => { 
-            :namespace => :string, :controller => :string, :action => :string, :format => :string } }
+            :namespace => :string, :controller => :string, :action => :string, :format => :string, :method => :string } }
     end
 
     # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}
     line_definition :completed do |line|
       line.footer = true
-      line.teaser = /\{:dispatch_time/
-      line.regexp = /\{\:dispatch_time=>(\d+\.\d+(?:e-?\d+)?), (?:\:after_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?(?:\:before_filters_time=>(\d+\.\d+(?:e-?\d+)?), )?\:action_time=>(\d+\.\d+(?:e-?\d+)?)\}/
-      line.captures << { :name => :dispatch_time,       :type => :duration } \
-                    << { :name => :after_filters_time,  :type => :duration } \
-                    << { :name => :before_filters_time, :type => :duration } \
-                    << { :name => :action_time,         :type => :duration }
+      line.regexp = /(\{.*\:dispatch_time\s*=>\s*\d+\.\d+.*\})/
+      line.captures << { :name => :times_hash, :type => :eval, :provides => {
+            :dispatch_time => :duration, :after_filters_time => :duration,
+            :before_filters_time => :duration, :action_time => :duration } }
     end
     
     REQUEST_CATEGORIZER = Proc.new do |request| 

data/lib/request_log_analyzer/line_definition.rb

@@ -66,6 +66,8 @@ module RequestLogAnalyzer
     
     alias :=~ :matches
 
+    # matches the line and converts the captured values using the request's
+    # convert_value function.
     def match_for(line, request, lineno = nil, parser = nil)
       if match_info = matches(line, lineno, parser)
         convert_captured_values(match_info[:captures], request)
@@ -74,11 +76,18 @@ module RequestLogAnalyzer
       end
     end
 
+    # Updates a captures hash using the converters specified in the request
+    # and handle the :provides option in the line definition.
     def convert_captured_values(values, request)
       value_hash = {}
       captures.each_with_index do |capture, index|
+        
+        # convert the value using the request convert_value function
         converted = request.convert_value(values[index], capture)
         value_hash[capture[:name]] ||= converted
+        
+        # Add items directly to the resulting hash from the converted value
+        # if it is a hash and they are set in the :provides hash for this line definition
         if converted.kind_of?(Hash) && capture[:provides].kind_of?(Hash)
           capture[:provides].each do |name, type|
             value_hash[name] ||= request.convert_value(converted[name], { :type => type })

data/lib/request_log_analyzer/request.rb

@@ -145,11 +145,11 @@ module RequestLogAnalyzer
     end
     
     def first_lineno
-      @lines.first[:lineno]
+      @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.min
     end
     
     def last_lineno
-      @lines.last[:lineno]
+      @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.max
     end
   end
 end

data/lib/request_log_analyzer/source/log_parser.rb

@@ -207,6 +207,10 @@ module RequestLogAnalyzer::Source
             warn(:unclosed_request, "Encountered header line (#{request_data[:line_definition].name.inspect}), but previous request was not closed!")
             @current_request = nil # remove all data that was parsed, skip next request as well.
           end
+          if footer_line?(request_data)
+            handle_request(@current_request, &block) # yield @current_request
+            @current_request = nil 
+          end
         else
           @current_request = @file_format.request(request_data)              
         end

data/lib/request_log_analyzer.rb

@@ -11,7 +11,7 @@ module RequestLogAnalyzer
   
   # The current version of request-log-analyzer.
   # This will be diplayed in output reports etc.  
-  VERSION = '1.2.1'
+  VERSION = "1.2.6"
   
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.

data/request-log-analyzer.gemspec

@@ -0,0 +1,36 @@
+Gem::Specification.new do |s|
+  s.name    = 'request-log-analyzer'
+  s.version = "1.2.6"
+  s.date    = "2009-09-02"
+  
+  s.rubyforge_project = 'r-l-a'
+  
+  s.bindir             = 'bin'
+  s.executables        = ['request-log-analyzer']
+  s.default_executable = 'request-log-analyzer'
+  
+  s.summary     = "A command line tool to analyze request logs for Rails, Merb and other application servers"
+  s.description = <<-eos
+    Request log analyzer's purpose is to find ot how your web application is being used and to focus your optimization efforts.
+    This tool will parse all requests in the application's log file and aggregate the information. Once it is finished parsing 
+    the log file(s), it will show the requests that take op most server time using various metrics. It can also insert all 
+    parsed request information into a database so you can roll your own analysis. It supports Rails- and Merb-based applications 
+    out of the box, but file formats of other applications can easily be supported by supplying an easy to write log file format 
+    definition.
+  eos
+  
+  s.rdoc_options << '--title' << s.name << '--main' << 'README.rdoc' << '--line-numbers' << '--inline-source'
+  s.extra_rdoc_files = ['README.rdoc']
+  
+  s.requirements << "To use the database inserter, ActiveRecord and an appropriate database adapter are required."
+  
+  s.add_development_dependency('rspec', '>= 1.2.4')
+  s.add_development_dependency('git',   '>= 1.1.0')
+  
+  s.authors  = ['Willem van Bergen', 'Bart ten Brinke']
+  s.email    = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
+  s.homepage = 'http://railsdoctors.com'
+
+  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/file_format/rails_development.rb spec/lib/macros.rb spec/fixtures/merb_prefixed.log tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/integration/command_line_usage_spec.rb spec/fixtures/decompression.log.bz2 lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb spec/fixtures/rails_unordered.log bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/matchers.rb spec/fixtures/test_order.log lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb spec/lib/testing_format.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb lib/request_log_analyzer.rb Rakefile spec/spec_helper.rb spec/unit/filter/filter_spec.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/test_language_combined.log spec/fixtures/decompression.tar.gz spec/unit/filter/field_filter_spec.rb spec/spec.opts lib/request_log_analyzer/aggregator/database.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb spec/unit/controller/controller_spec.rb lib/request_log_analyzer/output.rb spec/lib/helpers.rb spec/fixtures/rails_1x.log spec/lib/mocks.rb spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log lib/request_log_analyzer/source/database.rb spec/unit/aggregator/database_spec.rb tasks/github-gem.rake lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log spec/fixtures/syslog_1x.log LICENSE spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
+  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/aggregator/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
+end

data/spec/fixtures/header_and_footer.log

@@ -0,0 +1,6 @@
+start!
+this is a header and footer line
+blah
+
+this is a header and footer line
+blah

data/spec/fixtures/merb_prefixed.log

@@ -0,0 +1,9 @@
+Aug 31 18:35:23 typekit-web001 merb:  ~
+Aug 31 18:35:23 typekit-web001 merb: cache: [GET /] miss
+Aug 31 18:35:24 typekit-web001 merb:  ~ Started request handling: Mon Aug 31 18:35:25 +0000 2009
+Aug 31 18:35:24 typekit-web001 merb:  ~ Routed to: {"action"=>"index", "controller"=>"home"}
+Aug 31 18:35:24 typekit-web001 merb:  ~ Params: {"action"=>"index", "controller"=>"home"}
+Aug 31 18:35:24 typekit-web001 merb:  ~ In repository block default
+Aug 31 18:35:24 typekit-web001 merb:  ~ (0.000000) SELECT `id`, `created_at`, `updated_at`, `braintree_vault_id`, `cancelled_at` FROM `accounts` WHERE (`id` IN (6214)) ORDER BY `id`
+Aug 31 18:35:24 typekit-web001 merb:  ~ Redirecting to: /plans (302)
+Aug 31 18:35:24 typekit-web001 merb:  ~ {:after_filters_time=>0.0, :before_filters_time=>0.0, :dispatch_time=>0.012001, :action_time=>0.012001}

data/spec/integration/command_line_usage_spec.rb

@@ -2,8 +2,6 @@ require File.dirname(__FILE__) + '/../spec_helper.rb'
 
 describe RequestLogAnalyzer, 'running from command line' do
   
-  include RequestLogAnalyzer::Spec::Helper
-
   before(:each) do
     cleanup_temp_files!
   end

data/spec/lib/{helper.rb → helpers.rb}

@@ -1,7 +1,5 @@
-module RequestLogAnalyzer::Spec::Helper
+module RequestLogAnalyzer::Spec::Helpers
   
-  include RequestLogAnalyzer::Spec::Mocks
-
   # Create or return a new TestingFormat
   def testing_format
     @testing_format ||= TestingFormat.new

data/spec/lib/macros.rb

@@ -0,0 +1,2 @@
+module RequestLogAnalyzer::Spec::Macros
+end

data/spec/lib/matchers.rb

@@ -0,0 +1,63 @@
+module RequestLogAnalyzer::Spec::Matchers
+  
+  class HasLineDefinition
+    
+    def initialize(line_type)
+      @line_type = line_type.to_sym
+      @captures  = []
+    end
+      
+    def capturing(*captures)
+      @captures += captures
+      return self
+    end
+    
+    def matches?(file_format)
+      if file_format.new.line_definitions.include?(@line_type)
+        ld = file_format.new.line_definitions[@line_type]
+        @captures.all? { |c| ld.captures.include?(c) }
+      else
+        false
+      end
+    end
+  end
+  
+  class Parse
+    def initialize(line)
+      @line     = line
+      @captures = nil
+    end
+    
+    def failure_message
+      message = "expected to parse the provided line"
+      if @found_captures
+        message << "with captures #{@captures.inspect}, but captured #{@found_captures.inspect}"
+      end
+      return message
+    end
+    
+    def capturing(*captures)
+      @captures = captures
+      return self
+    end
+    
+    def matches?(line_definition)
+      hash = line_definition.matches(@line)
+      if hash
+        @found_captures = hash[:captures].to_a
+        @captures.nil? ? true : @found_captures.eql?(@captures)
+      else
+        return false
+      end
+    end
+  end
+  
+  def have_line_definition(line_type)
+    return HasLineDefinition.new(line_type)
+  end
+  
+  def parse(line)
+    Parse.new(line)
+  end
+  
+end

data/spec/lib/mocks.rb

@@ -38,10 +38,21 @@ module RequestLogAnalyzer::Spec::Mocks
     output.stub!(:title)
     output.stub!(:line)
     output.stub!(:with_style)    
-    output.stub!(:table) { yield [] }
+    output.stub!(:table).and_yield([])
     output.stub!(:io).and_return(mock_io)
     return output
   end
   
+  def mock_migrator
+    table_creator = mock('ActiveRecord table creator')
+    table_creator.stub!(:column)
+    
+    migrator = mock('ActiveRecord::Base.connection for migrations')
+    migrator.stub!(:add_index)
+    migrator.stub!(:create_table).and_yield(table_creator).and_return(true)
+    migrator.stub!(:table_creator).and_return(table_creator)
+    return migrator
+  end
+  
   
 end

data/spec/lib/testing_format.rb

@@ -27,6 +27,12 @@ class TestingFormat < RequestLogAnalyzer::FileFormat::Base
     line.captures = [{ :name => :request_no, :type => :integer }]
   end
   
+  format_definition.combined do |line|
+    line.header = true
+    line.footer = true
+    line.regexp = /this is a header and footer line/
+  end
+  
   report do |analyze|
     analyze.frequency :test_capture, :title => 'What is testing exactly?'
   end

data/spec/spec.opts

@@ -0,0 +1,3 @@
+--colour
+--format specdoc
+--reverse

data/spec/spec_helper.rb

@@ -2,14 +2,23 @@ $:.reject! { |e| e.include? 'TextMate' }
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'rubygems'
-require 'spec'
+require 'spec/autorun'
 require 'request_log_analyzer'
 
 module RequestLogAnalyzer::Spec
 end
 
-require File.dirname(__FILE__) + '/lib/testing_format'
-require File.dirname(__FILE__) + '/lib/mocks'
-require File.dirname(__FILE__) + '/lib/helper'
+# Include all files in the spec_helper directory
+Dir[File.dirname(__FILE__) + "/lib/**/*.rb"].each do |file| 
+  require file 
+end
 
 Dir.mkdir("#{File.dirname(__FILE__)}/../tmp") unless File.exist?("#{File.dirname(__FILE__)}/../tmp")
+
+Spec::Runner.configure do |config|
+  config.include RequestLogAnalyzer::Spec::Matchers  
+  config.include RequestLogAnalyzer::Spec::Mocks
+  config.include RequestLogAnalyzer::Spec::Helpers  
+  
+  config.extend RequestLogAnalyzer::Spec::Macros  
+end