request-log-analyzer 1.2.1 → 1.2.3

data/Rakefile

@@ -1,3 +1,3 @@
 Dir[File.dirname(__FILE__) + "/tasks/*.rake"].each { |file| load(file) }
  
-task :default => :spec
+task :default => "spec:fancy"

data/lib/request_log_analyzer/aggregator/database.rb

@@ -16,10 +16,13 @@ module RequestLogAnalyzer::Aggregator
   # created to log all parse warnings.
   class Database < Base
 
+    attr_reader :request_class, :request_count, :orm_module, :warning_class
+
     # Establishes a connection to the database and creates the necessary database schema for the
     # current file format
     def prepare
-      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
+      initialize_orm_module!
+      establish_database_connection!
       File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
       create_database_schema!
     end
@@ -28,9 +31,9 @@ module RequestLogAnalyzer::Aggregator
     # This will create a record in the requests table and create a record for every line that has been parsed,
     # in which the captured values will be stored.
     def aggregate(request)
-      @request_object = @request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
+      @request_object = request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
       request.lines.each do |line|
-        class_columns = @orm_module.const_get("#{line[:line_type]}_line".classify).column_names.reject { |column| ['id'].include?(column) }
+        class_columns = orm_module.const_get("#{line[:line_type]}_line".classify).column_names.reject { |column| ['id'].include?(column) }
         attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
         @request_object.send("#{line[:line_type]}_lines").build(attributes)
       end
@@ -41,13 +44,13 @@ module RequestLogAnalyzer::Aggregator
     
     # Finalizes the aggregator by closing the connection to the database
     def finalize
-      @request_count = @orm_module::Request.count
-      ActiveRecord::Base.remove_connection
+      @request_count = orm_module::Request.count
+      remove_database_connection!
     end
     
     # Records w warining in the warnings table.
     def warning(type, message, lineno)
-      @orm_module::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
+      warning_class.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
     end
     
     # Prints a short report of what has been inserted into the database
@@ -63,46 +66,72 @@ module RequestLogAnalyzer::Aggregator
     
     protected 
     
+    # Create a module and a default subclass of ActiveRecord::Base on which to establish the database connection
+    def initialize_orm_module!
+      
+      # Create a Database module in the file format if it does not yet exists
+      file_format.class.const_set('Database', Module.new) unless file_format.class.const_defined?('Database')
+      @orm_module = file_format.class.const_get('Database')
+      
+      # Register the base activerecord class
+      unless orm_module.const_defined?('Base')
+        orm_base_class = Class.new(ActiveRecord::Base)
+        orm_base_class.abstract_class = true
+        orm_module.const_set('Base', orm_base_class)
+      end
+    end    
+    
+    # Established a connection with the database for this session
+    def establish_database_connection!
+      orm_module::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
+      ActiveRecord::Migration.class_eval("def self.connection; #{@orm_module.to_s}::Base.connection; end ")
+    end    
+    
+    def remove_database_connection!
+      ActiveRecord::Migration.class_eval("def self.connection; ActiveRecord::Base.connection; end ")      
+      orm_module::Base.remove_connection
+    end
+    
     # This function creates a database table for a given line definition.
     # It will create a field for every capture in the line, and adds a lineno field to indicate at
     # what line in the original file the line was found, and a request_id to link lines related
     # to the same request. It will also create an index in the request_id field to speed up queries.
-    def create_database_table(name, definition)
-      ActiveRecord::Migration.verbose = options[:debug]
-      ActiveRecord::Migration.create_table("#{name}_lines") do |t|
+    def create_database_table(definition)
+      ActiveRecord::Migration.verbose = options[:debug] # keep it down a notch please
+      ActiveRecord::Migration.create_table("#{definition.name}_lines") do |t|
+        
+        # Add default fields for every line type
         t.column(:request_id, :integer)
         t.column(:lineno, :integer)
+        
         definition.captures.each do |capture|
-
           # Add a field for every capture
           t.column(capture[:name], column_type(capture[:type]))
-          
-          # If the capture provides other field as well, create them too
-          if capture[:provides].kind_of?(Hash)
-            capture[:provides].each do |field, field_type|
-              t.column(field, column_type(field_type))
-            end
-          end
+
+          # If the capture provides other field as well, create columns for them, too
+          capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
         end
       end
-      ActiveRecord::Migration.add_index("#{name}_lines", [:request_id])
+      
+      # Create an index on the request_id column to support faster querying
+      ActiveRecord::Migration.add_index("#{definition.name}_lines", [:request_id])
     end
     
     # Creates an ActiveRecord class for a given line definition.
     # A subclass of ActiveRecord::Base is created and an association with the Request class is
     # created using belongs_to / has_many. This association will later be used to create records
     # in the corresponding table. This table should already be created before this method is called.
-    def create_activerecord_class(name, definition)
-      class_name = "#{name}_line".camelize
-      klass = Class.new(ActiveRecord::Base)
+    def create_activerecord_class(definition)
+      class_name = "#{definition.name}_line".camelize
+      klass = Class.new(orm_module::Base)
       klass.send(:belongs_to, :request)
       
       definition.captures.each do |capture|
         klass.send(:serialize, capture[:name], Hash) if capture[:provides]
       end
       
-      @orm_module.const_set(class_name, klass) unless @orm_module.const_defined?(class_name)
-      @request_class.send(:has_many, "#{name}_lines".to_sym)
+      orm_module.const_set(class_name, klass) unless orm_module.const_defined?(class_name)
+      request_class.send(:has_many, "#{definition.name}_lines".to_sym)
     end    
     
     # Creates a requests table, in which a record is created for every request. It also creates an
@@ -114,8 +143,8 @@ module RequestLogAnalyzer::Aggregator
         t.integer :last_lineno
       end    
       
-      @orm_module.const_set('Request', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Request')     
-      @request_class = @orm_module.const_get('Request')
+      orm_module.const_set('Request', Class.new(orm_module::Base)) unless orm_module.const_defined?('Request')     
+      @request_class = orm_module.const_get('Request')
     end
 
     # Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
@@ -127,25 +156,19 @@ module RequestLogAnalyzer::Aggregator
         t.integer :lineno          
       end    
       
-      @orm_module.const_set('Warning', Class.new(ActiveRecord::Base)) unless @orm_module.const_defined?('Warning')
+      orm_module.const_set('Warning', Class.new(orm_module::Base)) unless orm_module.const_defined?('Warning')
+      @warning_class = orm_module.const_get('Warning')
     end
     
     # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 
     # file format definition. These ORM classes will later be used to create records in the database.
     def create_database_schema!
-      
-      if file_format.class.const_defined?('Database')
-        @orm_module = file_format.class.const_get('Database')
-      else
-        @orm_module = file_format.class.const_set('Database', Module.new)
-      end
-
       create_request_table_and_class
       create_warning_table_and_class
       
       file_format.line_definitions.each do |name, definition|
-        create_database_table(name, definition)
-        create_activerecord_class(name, definition)
+        create_database_table(definition)
+        create_activerecord_class(definition)
       end
     end
     
@@ -154,6 +177,7 @@ module RequestLogAnalyzer::Aggregator
     def column_type(type_indicator)
       case type_indicator
       when :eval;      :text
+      when :hash;      :text
       when :text;      :text
       when :string;    :string
       when :sec;       :double

data/lib/request_log_analyzer/controller.rb

@@ -191,14 +191,11 @@ module RequestLogAnalyzer
     def run!
       
       @aggregators.each { |agg| agg.prepare }
+      install_signal_handlers
       
-      begin
-        @source.each_request do |request|
-          aggregate_request(filter_request(request))
-        end
-      rescue Interrupt => e
-        handle_progress(:interrupted)
-        puts "Caught interrupt! Stopped parsing."
+      @source.each_request do |request|
+        aggregate_request(filter_request(request))
+        break if @interrupted
       end
 
       @aggregators.each { |agg| agg.finalize }
@@ -219,5 +216,13 @@ module RequestLogAnalyzer
       end
     end
     
+    def install_signal_handlers
+      Signal.trap("INT") do
+        handle_progress(:interrupted)
+        puts "Caught interrupt! Stopping parsing..."
+        @interrupted = true
+      end
+    end
+    
   end
 end

data/lib/request_log_analyzer/line_definition.rb

@@ -66,6 +66,8 @@ module RequestLogAnalyzer
     
     alias :=~ :matches
 
+    # matches the line and converts the captured values using the request's
+    # convert_value function.
     def match_for(line, request, lineno = nil, parser = nil)
       if match_info = matches(line, lineno, parser)
         convert_captured_values(match_info[:captures], request)
@@ -74,11 +76,18 @@ module RequestLogAnalyzer
       end
     end
 
+    # Updates a captures hash using the converters specified in the request
+    # and handle the :provides option in the line definition.
     def convert_captured_values(values, request)
       value_hash = {}
       captures.each_with_index do |capture, index|
+        
+        # convert the value using the request convert_value function
         converted = request.convert_value(values[index], capture)
         value_hash[capture[:name]] ||= converted
+        
+        # Add items directly to the resulting hash from the converted value
+        # if it is a hash and they are set in the :provides hash for this line definition
         if converted.kind_of?(Hash) && capture[:provides].kind_of?(Hash)
           capture[:provides].each do |name, type|
             value_hash[name] ||= request.convert_value(converted[name], { :type => type })

data/lib/request_log_analyzer/source/log_parser.rb

@@ -207,6 +207,10 @@ module RequestLogAnalyzer::Source
             warn(:unclosed_request, "Encountered header line (#{request_data[:line_definition].name.inspect}), but previous request was not closed!")
             @current_request = nil # remove all data that was parsed, skip next request as well.
           end
+          if footer_line?(request_data)
+            handle_request(@current_request, &block) # yield @current_request
+            @current_request = nil 
+          end
         else
           @current_request = @file_format.request(request_data)              
         end

data/spec/fixtures/header_and_footer.log

@@ -0,0 +1,6 @@
+start!
+this is a header and footer line
+blah
+
+this is a header and footer line
+blah

data/spec/integration/command_line_usage_spec.rb

@@ -2,8 +2,6 @@ require File.dirname(__FILE__) + '/../spec_helper.rb'
 
 describe RequestLogAnalyzer, 'running from command line' do
   
-  include RequestLogAnalyzer::Spec::Helper
-
   before(:each) do
     cleanup_temp_files!
   end

data/spec/lib/{helper.rb → helpers.rb}

@@ -1,7 +1,5 @@
-module RequestLogAnalyzer::Spec::Helper
+module RequestLogAnalyzer::Spec::Helpers
   
-  include RequestLogAnalyzer::Spec::Mocks
-
   # Create or return a new TestingFormat
   def testing_format
     @testing_format ||= TestingFormat.new

data/spec/lib/macros.rb

@@ -0,0 +1,2 @@
+module RequestLogAnalyzer::Spec::Macros
+end

data/spec/lib/matchers.rb

@@ -0,0 +1,63 @@
+module RequestLogAnalyzer::Spec::Matchers
+  
+  class HasLineDefinition
+    
+    def initialize(line_type)
+      @line_type = line_type.to_sym
+      @captures  = []
+    end
+      
+    def capturing(*captures)
+      @captures += captures
+      return self
+    end
+    
+    def matches?(file_format)
+      if file_format.new.line_definitions.include?(@line_type)
+        ld = file_format.new.line_definitions[@line_type]
+        @captures.all? { |c| ld.captures.include?(c) }
+      else
+        false
+      end
+    end
+  end
+  
+  class Parse
+    def initialize(line)
+      @line     = line
+      @captures = nil
+    end
+    
+    def failure_message
+      message = "expected to parse the provided line"
+      if @found_captures
+        message << "with captures #{@captures.inspect}, but captured #{@found_captures.inspect}"
+      end
+      return message
+    end
+    
+    def capturing(*captures)
+      @captures = captures
+      return self
+    end
+    
+    def matches?(line_definition)
+      hash = line_definition.matches(@line)
+      if hash
+        @found_captures = hash[:captures].to_a
+        @captures.nil? ? true : @found_captures.eql?(@captures)
+      else
+        return false
+      end
+    end
+  end
+  
+  def have_line_definition(line_type)
+    return HasLineDefinition.new(line_type)
+  end
+  
+  def parse(line)
+    Parse.new(line)
+  end
+  
+end

data/spec/lib/testing_format.rb

@@ -27,6 +27,12 @@ class TestingFormat < RequestLogAnalyzer::FileFormat::Base
     line.captures = [{ :name => :request_no, :type => :integer }]
   end
   
+  format_definition.combined do |line|
+    line.header = true
+    line.footer = true
+    line.regexp = /this is a header and footer line/
+  end
+  
   report do |analyze|
     analyze.frequency :test_capture, :title => 'What is testing exactly?'
   end

data/spec/spec.opts

@@ -0,0 +1,3 @@
+--colour
+--format specdoc
+--reverse

data/spec/spec_helper.rb

@@ -2,14 +2,23 @@ $:.reject! { |e| e.include? 'TextMate' }
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'rubygems'
-require 'spec'
+require 'spec/autorun'
 require 'request_log_analyzer'
 
 module RequestLogAnalyzer::Spec
 end
 
-require File.dirname(__FILE__) + '/lib/testing_format'
-require File.dirname(__FILE__) + '/lib/mocks'
-require File.dirname(__FILE__) + '/lib/helper'
+# Include all files in the spec_helper directory
+Dir[File.dirname(__FILE__) + "/lib/**/*.rb"].each do |file| 
+  require file 
+end
 
 Dir.mkdir("#{File.dirname(__FILE__)}/../tmp") unless File.exist?("#{File.dirname(__FILE__)}/../tmp")
+
+Spec::Runner.configure do |config|
+  config.include RequestLogAnalyzer::Spec::Matchers  
+  config.include RequestLogAnalyzer::Spec::Mocks
+  config.include RequestLogAnalyzer::Spec::Helpers  
+  
+  config.extend RequestLogAnalyzer::Spec::Macros  
+end

data/spec/unit/aggregator/database_spec.rb

@@ -0,0 +1,208 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Aggregator::Database do
+
+  before(:each) do
+    log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format)
+    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => ':memory:')
+    
+    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, { :regexp   => /Testing (\w+), tries\: (\d+)/,
+                          :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer },
+                            { :name => :evaluated, :type => :hash, :provides => {:evaluated_field => :duration} }]})    
+  end
+  
+  # The prepare method is called before the parsing starts. It should establish a connection
+  # to a database that is suitable for inserting requests later on.
+  describe '#prepare' do
+
+    before(:each) do
+      @database_inserter.stub!(:establish_database_connection!)      
+      @database_inserter.stub!(:create_database_schema!)
+    end
+
+    it 'should establish the database connection' do
+      @database_inserter.should_receive(:establish_database_connection!)      
+      @database_inserter.prepare
+    end
+    
+    it 'should create the database schema during preparation' do
+      @database_inserter.should_receive(:create_database_schema!)
+      @database_inserter.prepare
+    end
+  end
+  
+  # The create_database_table method should create a database table according to the line definition,
+  # so that parsed lines can be stored in it later on.
+  describe '#create_database_table' do
+    
+    before(:each) do
+      @table_creator = mock('create_table block')
+      @table_creator.stub!(:column)
+      ActiveRecord::Migration.stub!(:create_table).and_yield(@table_creator)
+      ActiveRecord::Migration.stub!(:add_index)
+    end
+    
+    it "should create a table based on the line type name" do
+      ActiveRecord::Migration.should_receive(:create_table).with('test_lines')
+      @database_inserter.send(:create_database_table, @line_definition)
+    end    
+    
+    it "should create an index on the request_id field" do
+      ActiveRecord::Migration.should_receive(:add_index).with('test_lines', [:request_id])
+      @database_inserter.send(:create_database_table, @line_definition)
+    end
+    
+    it "should create a request_id field to link the requests together" do
+      @table_creator.should_receive(:column).with(:request_id, :integer)
+      @database_inserter.send(:create_database_table, @line_definition)
+    end
+    
+    it "should create a lineno field to save the location of the line in the original file" do
+      @table_creator.should_receive(:column).with(:lineno, :integer)
+      @database_inserter.send(:create_database_table, @line_definition)
+    end
+    
+    it "should create a field of the correct type for every defined field" do
+      @table_creator.should_receive(:column).with(:what, :string)
+      @table_creator.should_receive(:column).with(:tries, :integer)
+      # :hash capture type should map on a :text field type
+      @table_creator.should_receive(:column).with(:evaluated, :text) 
+      @database_inserter.send(:create_database_table, @line_definition)      
+    end
+    
+    it "should create a field of the correct type for every provided field" do
+      # :duration capture type should map on a :double field type
+      @table_creator.should_receive(:column).with(:evaluated_field, :double) 
+      @database_inserter.send(:create_database_table, @line_definition)   
+    end
+  end
+  
+  describe '#create_activerecord_class' do
+    before(:each) do
+      # Make sure the ORM module exists
+      @database_inserter.send(:initialize_orm_module!)     
+      
+      # Mockthe request ORM class
+      @request_class = mock('Request ActiveRecord::Base class')
+      @request_class.stub!(:has_many)
+      @database_inserter.stub!(:request_class).and_return(@request_class)
+    end
+    
+    it "should register the constant for the line type AR class" do
+      @database_inserter.send(:create_activerecord_class, @line_definition)
+      @database_inserter.orm_module.const_defined?('TestLine').should be_true
+    end
+    
+    it "should create a class that inherits from ActiveRecord::Base and the base class of the ORM module" do
+      @database_inserter.send(:create_activerecord_class, @line_definition)
+      @database_inserter.orm_module.const_get('TestLine').ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
+    end    
+    
+    describe 'defining the new ORM class' do
+      before(:each) do
+        # Mock the newly created ORM class for the test_line
+        @orm_class = mock('Line ActiveRecord::Base class')
+        @orm_class.stub!(:belongs_to)
+        @orm_class.stub!(:serialize)
+        Class.stub!(:new).and_return(@orm_class)
+      end
+
+      it "should create a has_many relation on the Request class" do
+        @request_class.should_receive(:has_many).with(:test_lines)
+        @database_inserter.send(:create_activerecord_class, @line_definition)
+      end
+    
+      it "should create a belongs_to relation to the request class" do
+        @orm_class.should_receive(:belongs_to).with(:request)
+        @database_inserter.send(:create_activerecord_class, @line_definition)            
+      end
+    
+      it "should serialize the :evaluate field into the database" do
+        @orm_class.should_receive(:serialize).with(:evaluated, Hash)
+        @database_inserter.send(:create_activerecord_class, @line_definition)            
+      end    
+    end
+  end
+  
+  describe '#create_database_schema!' do
+  
+    before(:each) do
+      @line_type_cnt = testing_format.line_definitions.length
+
+      #Make sure no actual migrations occur
+      ActiveRecord::Migration.stub!(:add_index)
+      ActiveRecord::Migration.stub!(:create_table)
+      
+      # Stub the expected method calls for the preparation
+      @database_inserter.stub!(:create_database_table)
+      @database_inserter.stub!(:create_activerecord_class) 
+
+      
+      # Make sure the ORM module exists
+      @database_inserter.send(:initialize_orm_module!)
+    end
+
+    it "should create a requests table to join request lines" do
+      ActiveRecord::Migration.should_receive(:create_table).with("requests")
+      @database_inserter.send :create_database_schema!
+    end
+        
+    it "should create a Request class inheriting from ActiveRecord and the base class of the ORM module" do
+      @database_inserter.send :create_database_schema!
+      @database_inserter.request_class.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
+    end
+    
+    it "should create a warnings table for logging parse warnings" do
+      ActiveRecord::Migration.should_receive(:create_table).with("warnings")
+      @database_inserter.send :create_database_schema!
+    end
+
+    it "should create a Warnng class inheriting from ActiveRecord and the base class of the ORM module" do
+      @database_inserter.send :create_database_schema!
+      @database_inserter.warning_class.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
+    end
+  
+    it "should create a table for every line type" do
+      @database_inserter.should_receive(:create_database_table).with(an_instance_of(RequestLogAnalyzer::LineDefinition)).exactly(@line_type_cnt).times
+      @database_inserter.send :create_database_schema!
+    end
+    
+    it "should create a ORM for every line type" do
+      @database_inserter.should_receive(:create_activerecord_class).with(an_instance_of(RequestLogAnalyzer::LineDefinition)).exactly(@line_type_cnt).times
+      @database_inserter.send :create_database_schema!
+    end
+  end
+  
+  describe '#aggregate' do
+    before(:each) do
+      @database_inserter.prepare
+
+      @incomplete_request = testing_format.request( {:line_type => :first, :request_no => 564})
+      @completed_request = testing_format.request( {:line_type => :first, :request_no  => 564}, 
+                            {:line_type => :test, :test_capture => "awesome"},
+                            {:line_type => :test, :test_capture => "indeed"}, 
+                            {:line_type => :eval, :evaluated => { :greating => 'howdy'}, :greating => 'howdy' }, 
+                            {:line_type => :last, :request_no   => 564})    
+    end   
+    
+    it "should insert a record in the request table" do
+      TestingFormat::Database::Request.count.should == 0
+      @database_inserter.aggregate(@incomplete_request)
+      TestingFormat::Database::Request.count.should == 1
+    end
+
+    it "should insert records in all relevant line tables" do
+      @database_inserter.aggregate(@completed_request)
+      request = TestingFormat::Database::Request.first
+      request.should have(2).test_lines
+      request.should have(1).first_lines  
+      request.should have(1).eval_lines     
+      request.should have(1).last_lines    
+    end
+
+    it "should log a warning in the warnings table" do
+      TestingFormat::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
+      @database_inserter.warning(:test_warning, "Testing the warning system", 12)
+    end    
+  end
+end