wvanbergen-request-log-analyzer 1.2.9 → 1.3.0

data/lib/request_log_analyzer/aggregator/database.rb

@@ -1,220 +0,0 @@
-require 'rubygems'
-require 'activerecord'
-
-module RequestLogAnalyzer::Aggregator
-
-  # The database aggregator will create an SQLite3 database with all parsed request information.
-  #
-  # The prepare method will create a database schema according to the file format definitions.
-  # It will also create ActiveRecord::Base subclasses to interact with the created tables. 
-  # Then, the aggregate method will be called for every parsed request. The information of
-  # these requests is inserted into the tables using the ActiveRecord classes.
-  #
-  # A requests table will be created, in which a record is inserted for every parsed request.
-  # For every line type, a separate table will be created with a request_id field to point to
-  # the request record, and a field for every parsed value. Finally, a warnings table will be
-  # created to log all parse warnings.
-  class Database < Base
-
-    attr_reader :request_class, :request_count, :orm_module, :warning_class
-
-    # Establishes a connection to the database and creates the necessary database schema for the
-    # current file format
-    def prepare
-      initialize_orm_module!
-      establish_database_connection!
-      File.unlink(options[:database]) if File.exist?(options[:database]) # TODO: keep old database?
-      create_database_schema!
-    end
-    
-    # Aggregates a request into the database
-    # This will create a record in the requests table and create a record for every line that has been parsed,
-    # in which the captured values will be stored.
-    def aggregate(request)
-      @request_object = request_class.new(:first_lineno => request.first_lineno, :last_lineno => request.last_lineno)
-      request.lines.each do |line|
-        class_columns = orm_module.const_get("#{line[:line_type]}_line".classify).column_names.reject { |column| ['id'].include?(column) }
-        attributes = Hash[*line.select { |(k, v)| class_columns.include?(k.to_s) }.flatten]
-        @request_object.send("#{line[:line_type]}_lines").build(attributes)
-      end
-      @request_object.save!
-    rescue SQLite3::SQLException => e
-      raise Interrupt, e.message
-    end
-    
-    # Finalizes the aggregator by closing the connection to the database
-    def finalize
-      @request_count = orm_module::Request.count
-      remove_database_connection!
-      deinitialize_orm_module!
-    end
-    
-    # Records w warining in the warnings table.
-    def warning(type, message, lineno)
-      warning_class.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
-    end
-    
-    # Prints a short report of what has been inserted into the database
-    def report(output)
-      output.title('Request database created')
-      
-      output <<  "A database file has been created with all parsed request information.\n"
-      output <<  "#{@request_count} requests have been added to the database.\n"      
-      output <<  "To execute queries on this database, run the following command:\n"
-      output <<  output.colorize("  $ sqlite3 #{options[:database]}\n", :bold)
-      output << "\n"
-    end
-    
-    # Retreives the connection that is used for the database inserter
-    def connection
-      orm_module::Base.connection
-    end    
-    
-    protected 
-    
-    # Create a module and a default subclass of ActiveRecord::Base on which to establish the database connection
-    def initialize_orm_module!
-      
-      # Create a Database module in the file format if it does not yet exists
-      file_format.class.const_set('Database', Module.new) unless file_format.class.const_defined?('Database')
-      @orm_module = file_format.class.const_get('Database')
-      
-      # Register the base activerecord class
-      unless orm_module.const_defined?('Base')
-        orm_base_class = Class.new(ActiveRecord::Base)
-        orm_base_class.abstract_class = true
-        orm_module.const_set('Base', orm_base_class)
-      end
-    end
-
-    # Deinitializes the ORM module and the ActiveRecord::Base subclass.
-    def deinitialize_orm_module!
-      file_format.class.send(:remove_const, 'Database') if file_format.class.const_defined?('Database')
-      @orm_module = nil
-    end
-
-    # Established a connection with the database for this session
-    def establish_database_connection!
-      orm_module::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
-      #ActiveRecord::Migration.class_eval("def self.connection; #{@orm_module.to_s}::Base.connection; end ")
-    end
-
-    def remove_database_connection!
-      #ActiveRecord::Migration.class_eval("def self.connection; ActiveRecord::Base.connection; end ")      
-      orm_module::Base.remove_connection
-    end
-    
-    # This function creates a database table for a given line definition.
-    # It will create a field for every capture in the line, and adds a lineno field to indicate at
-    # what line in the original file the line was found, and a request_id to link lines related
-    # to the same request. It will also create an index in the request_id field to speed up queries.
-    def create_database_table(definition)
-      connection.create_table("#{definition.name}_lines") do |t|
-        
-        # Add default fields for every line type
-        t.column(:request_id, :integer)
-        t.column(:lineno, :integer)
-        
-        definition.captures.each do |capture|
-          # Add a field for every capture
-          t.column(capture[:name], column_type(capture[:type]))
-
-          # If the capture provides other field as well, create columns for them, too
-          capture[:provides].each { |field, field_type| t.column(field, column_type(field_type)) } if capture[:provides].kind_of?(Hash)
-        end
-      end
-      
-      # Create an index on the request_id column to support faster querying
-      connection.add_index("#{definition.name}_lines", [:request_id])
-    end
-    
-    # Creates an ActiveRecord class for a given line definition.
-    # A subclass of ActiveRecord::Base is created and an association with the Request class is
-    # created using belongs_to / has_many. This association will later be used to create records
-    # in the corresponding table. This table should already be created before this method is called.
-    def create_activerecord_class(definition)
-      class_name = "#{definition.name}_line".camelize
-      klass = Class.new(orm_module::Base)
-      klass.send(:belongs_to, :request)
-      
-      definition.captures.select { |c| c.has_key?(:provides) }.each do |capture|
-        klass.send(:serialize, capture[:name], Hash)
-      end
-      
-      orm_module.const_set(class_name, klass) unless orm_module.const_defined?(class_name)
-      request_class.send(:has_many, "#{definition.name}_lines".to_sym)
-    end
-
-    # Creates a requests table, in which a record is created for every parsed request. 
-    # It also creates an ActiveRecord::Base class to communicate with this table.
-    def create_request_table_and_class
-      connection.create_table("requests") do |t|
-        t.column :first_lineno, :integer
-        t.column :last_lineno,  :integer
-      end    
-      
-      orm_module.const_set('Request', Class.new(orm_module::Base)) unless orm_module.const_defined?('Request')     
-      @request_class = orm_module.const_get('Request')
-    end
-
-    # Creates a sources table, in which a record is created for every file that is parsed. 
-    # It also creates an ActiveRecord::Base ORM class for the table.
-    def create_source_table_and_class
-      connection.create_table('sources') do |t|
-        t.column :filename, :string
-        t.column :mtime,    :datetime
-        t.column :filesize, :integer
-      end
-      
-      orm_module.const_set('Source', Class.new(orm_module::Base)) unless orm_module.const_defined?('Source')
-      @source_class = orm_module.const_get('Source')      
-    end
-
-    # Creates a warnings table and a corresponding Warning class to communicate with this table using ActiveRecord.
-    def create_warning_table_and_class
-      connection.create_table("warnings") do |t|
-        t.column  :warning_type, :string, :limit => 30, :null => false
-        t.column  :message, :string
-        t.column  :lineno, :integer
-      end    
-      
-      orm_module.const_set('Warning', Class.new(orm_module::Base)) unless orm_module.const_defined?('Warning')
-      @warning_class = orm_module.const_get('Warning')
-    end
-    
-    # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 
-    # file format definition. These ORM classes will later be used to create records in the database.
-    def create_database_schema!
-      create_source_table_and_class
-      create_request_table_and_class
-      create_warning_table_and_class
-      
-      file_format.line_definitions.each do |name, definition|
-        create_database_table(definition)
-        create_activerecord_class(definition)
-      end
-    end
-    
-    # Function to determine the column type for a field
-    # TODO: make more robust / include in file-format definition
-    def column_type(type_indicator)
-      case type_indicator
-      when :eval;      :text
-      when :hash;      :text
-      when :text;      :text
-      when :string;    :string
-      when :sec;       :double
-      when :msec;      :double
-      when :duration;  :double
-      when :float;     :double
-      when :double;    :double
-      when :integer;   :integer
-      when :int;       :int
-      when :timestamp; :datetime
-      when :datetime;  :datetime
-      when :date;      :date
-      else             :string
-      end
-    end
-  end
-end

data/spec/spec.opts

@@ -1,3 +0,0 @@
---colour
---format specdoc
---reverse

data/spec/unit/aggregator/database_spec.rb

@@ -1,245 +0,0 @@
-require File.dirname(__FILE__) + '/../../spec_helper'
-
-describe RequestLogAnalyzer::Aggregator::Database do
-
-  before(:all) do
-    log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format)
-    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => ':memory:')
-    
-    @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, { :regexp   => /Testing (\w+), tries\: (\d+)/,
-                          :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer },
-                            { :name => :evaluated, :type => :hash, :provides => {:evaluated_field => :duration} }]})                               
-  end
-  
-  # The prepare method is called before the parsing starts. It should establish a connection
-  # to a database that is suitable for inserting requests later on.
-  describe '#prepare' do
-
-    before(:each) do
-      @database_inserter.stub!(:initialize_orm_module!)
-      @database_inserter.stub!(:establish_database_connection!)
-      @database_inserter.stub!(:create_database_schema!)
-    end
-
-    it 'should create the ORM mdoule in which the classes can be created' do
-      @database_inserter.should_receive(:initialize_orm_module!)
-      @database_inserter.prepare
-    end
-
-    it 'should establish the database connection' do
-      @database_inserter.should_receive(:establish_database_connection!)
-      @database_inserter.prepare
-    end
-    
-    it 'should create the database schema during preparation' do
-      @database_inserter.should_receive(:create_database_schema!)
-      @database_inserter.prepare
-    end
-  end
-  
-  # The database inserter creates is own "Database" module within the file format 
-  # class to create all the classes that are needed.
-  describe '#initialize_orm_module!' do
-    
-    before(:all) { @database_inserter.send(:deinitialize_orm_module!) }
-    after(:all)  { @database_inserter.send(:initialize_orm_module!) }
-    
-    before(:each) { @database_inserter.send(:initialize_orm_module!) }
-    after(:each)  { @database_inserter.send(:deinitialize_orm_module!) }
-    
-    it "should create a Database module under the file format's class" do
-      testing_format.class.should be_const_defined('Database')
-    end
-    
-    it "should define a Base class in the Database module" do
-      testing_format.class::Database.should be_const_defined('Base')
-    end
-    
-    it "should create a ActiveRecord::Base class in the Database module" do
-      testing_format.class::Database::Base.ancestors.should include(ActiveRecord::Base)
-    end
-    
-  end
-  
-  # The create_database_table method should create a database table according to the line definition,
-  # so that parsed lines can be stored in it later on.
-  describe '#create_database_table' do
-    
-    before(:each) do
-      @connection = mock_migrator
-      @database_inserter.stub!(:connection).and_return(@connection)
-    end
-    
-    it "should create a table based on the line type name" do
-      @connection.should_receive(:create_table).with('test_lines')
-      @database_inserter.send(:create_database_table, @line_definition)
-    end    
-    
-    it "should create an index on the request_id field" do
-      @connection.should_receive(:add_index).with('test_lines', [:request_id])
-      @database_inserter.send(:create_database_table, @line_definition)
-    end
-    
-    it "should create a request_id field to link the requests together" do
-      @connection.table_creator.should_receive(:column).with(:request_id, :integer)
-      @database_inserter.send(:create_database_table, @line_definition)
-    end
-    
-    it "should create a lineno field to save the location of the line in the original file" do
-      @connection.table_creator.should_receive(:column).with(:lineno, :integer)
-      @database_inserter.send(:create_database_table, @line_definition)
-    end
-    
-    it "should create a field of the correct type for every defined field" do
-      @connection.table_creator.should_receive(:column).with(:what, :string)
-      @connection.table_creator.should_receive(:column).with(:tries, :integer)
-      # :hash capture type should map on a :text field type
-      @connection.table_creator.should_receive(:column).with(:evaluated, :text) 
-      @database_inserter.send(:create_database_table, @line_definition)      
-    end
-    
-    it "should create a field of the correct type for every provided field" do
-      # :duration capture type should map on a :double field type
-      @connection.table_creator.should_receive(:column).with(:evaluated_field, :double) 
-      @database_inserter.send(:create_database_table, @line_definition)   
-    end
-  end
-  
-  describe '#create_activerecord_class' do
-    before(:each) do
-      # Make sure the ORM module exists
-      @database_inserter.send(:initialize_orm_module!)     
-      
-      # Mockthe request ORM class
-      @request_class = mock('Request ActiveRecord::Base class')
-      @request_class.stub!(:has_many)
-      @database_inserter.stub!(:request_class).and_return(@request_class)
-    end
-    
-    it "should register the constant for the line type AR class" do
-      @database_inserter.send(:create_activerecord_class, @line_definition)
-      @database_inserter.orm_module.const_defined?('TestLine').should be_true
-    end
-    
-    it "should create a class that inherits from ActiveRecord::Base and the base class of the ORM module" do
-      @database_inserter.send(:create_activerecord_class, @line_definition)
-      @database_inserter.orm_module.const_get('TestLine').ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
-    end    
-    
-    describe 'defining the new ORM class' do
-      before(:each) do
-        # Mock the newly created ORM class for the test_line
-        @orm_class = mock('Line ActiveRecord::Base class')
-        @orm_class.stub!(:belongs_to)
-        @orm_class.stub!(:serialize)
-        Class.stub!(:new).and_return(@orm_class)
-      end
-
-      it "should create a has_many relation on the Request class" do
-        @request_class.should_receive(:has_many).with(:test_lines)
-        @database_inserter.send(:create_activerecord_class, @line_definition)
-      end
-    
-      it "should create a belongs_to relation to the request class" do
-        @orm_class.should_receive(:belongs_to).with(:request)
-        @database_inserter.send(:create_activerecord_class, @line_definition)            
-      end
-    
-      it "should serialize the :evaluate field into the database" do
-        @orm_class.should_receive(:serialize).with(:evaluated, Hash)
-        @database_inserter.send(:create_activerecord_class, @line_definition)            
-      end    
-    end
-  end
-  
-  describe '#create_database_schema!' do
-  
-    before(:each) do
-      @line_type_cnt = testing_format.line_definitions.length
-
-      @connection = mock_migrator
-      @database_inserter.stub!(:connection).and_return(@connection)      
-      
-      # Stub the expected method calls for the preparation
-      @database_inserter.stub!(:create_database_table)
-      @database_inserter.stub!(:create_activerecord_class) 
-
-      # Make sure the ORM module exists
-      @database_inserter.send(:initialize_orm_module!)
-    end
-
-    it "should create a requests table to join request lines" do
-      @connection.should_receive(:create_table).with("requests")
-      @database_inserter.send :create_database_schema!
-    end
-        
-    it "should create a Request class inheriting from ActiveRecord and the base class of the ORM module" do
-      @database_inserter.send :create_database_schema!
-      @database_inserter.request_class.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
-    end
-    
-    it "should create a warnings table for logging parse warnings" do
-      @connection.should_receive(:create_table).with("warnings")
-      @database_inserter.send :create_database_schema!
-    end
-
-    it "should create a Warning class inheriting from ActiveRecord and the base class of the ORM module" do
-      @database_inserter.send :create_database_schema!
-      @database_inserter.warning_class.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
-    end
-    
-    it "should create a sources table to track parsed files" do
-      @connection.should_receive(:create_table).with("sources")
-      @database_inserter.send :create_database_schema!
-    end    
-
-    it "should create a Source ORM class" do
-      @database_inserter.send :create_database_schema!
-      @database_inserter.orm_module::Source.ancestors.should include(ActiveRecord::Base, @database_inserter.orm_module::Base)
-    end    
-
-  
-    it "should create a table for every line type" do
-      @database_inserter.should_receive(:create_database_table).with(an_instance_of(RequestLogAnalyzer::LineDefinition)).exactly(@line_type_cnt).times
-      @database_inserter.send :create_database_schema!
-    end
-    
-    it "should create a ORM for every line type" do
-      @database_inserter.should_receive(:create_activerecord_class).with(an_instance_of(RequestLogAnalyzer::LineDefinition)).exactly(@line_type_cnt).times
-      @database_inserter.send :create_database_schema!
-    end
-  end
-  
-  describe '#aggregate' do
-    before(:each) do
-      @database_inserter.prepare
-
-      @incomplete_request = testing_format.request( {:line_type => :first, :request_no => 564})
-      @completed_request = testing_format.request( {:line_type => :first, :request_no  => 564}, 
-                            {:line_type => :test, :test_capture => "awesome"},
-                            {:line_type => :test, :test_capture => "indeed"}, 
-                            {:line_type => :eval, :evaluated => { :greating => 'howdy'}, :greating => 'howdy' }, 
-                            {:line_type => :last, :request_no   => 564})    
-    end   
-    
-    it "should insert a record in the request table" do
-      TestingFormat::Database::Request.count.should == 0
-      @database_inserter.aggregate(@incomplete_request)
-      TestingFormat::Database::Request.count.should == 1
-    end
-
-    it "should insert records in all relevant line tables" do
-      @database_inserter.aggregate(@completed_request)
-      request = TestingFormat::Database::Request.first
-      request.should have(2).test_lines
-      request.should have(1).first_lines  
-      request.should have(1).eval_lines     
-      request.should have(1).last_lines    
-    end
-
-    it "should log a warning in the warnings table" do
-      TestingFormat::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
-      @database_inserter.warning(:test_warning, "Testing the warning system", 12)
-    end    
-  end
-end