ngmoco-request-log-analyzer 1.4.2

data/spec/lib/macros.rb

@@ -0,0 +1,18 @@
+module RequestLogAnalyzer::Spec::Macros
+
+  def test_databases
+    require 'yaml'
+    hash = YAML.load(File.read("#{File.dirname(__FILE__)}/../database.yml"))
+    hash.inject({}) { |res, (name, h)| res[name] = h.map { |(k,v)| "#{k}=#{v}" }.join(';'); res  }
+  end
+
+  # Create or return a new TestingFormat
+  def testing_format
+    @testing_format ||= TestingFormat.create
+  end
+
+  def default_orm_class_names
+    ['Warning', 'Request', 'Source']
+  end
+
+end

data/spec/lib/matchers.rb

@@ -0,0 +1,77 @@
+module RequestLogAnalyzer::Spec::Matchers
+
+  class HasLineDefinition
+
+    def initialize(line_type)
+      @line_type = line_type.to_sym
+      @captures  = []
+    end
+
+    def capturing(*captures)
+      @captures += captures
+      return self
+    end
+
+    def matches?(file_format)
+      file_format = file_format.create if file_format.kind_of?(Class)
+      if ld = file_format.line_definitions[@line_type]
+        @captures.all? { |c| ld.captures.map { |cd| cd[:name] }.include?(c) }
+      else
+        false
+      end
+    end
+  end
+
+  class ParseLine
+    
+    def initialize(line)
+      @line      = line
+      @captures  = {}
+      @line_type = nil
+    end
+
+    def failure_message
+      @failure_message
+    end
+
+    def as(line_type)
+      @line_type = line_type
+      return self
+    end
+    
+    def and_capture(captures)
+      @captures = captures
+      return self
+    end
+
+    def fail(message)
+      @failure_message = message
+      return false
+    end
+
+    def matches?(file_format)
+      if @line_hash = file_format.parse_line(@line)
+        if @line_type.nil? || @line_hash[:line_definition].name == @line_type
+          @request   = file_format.request(@line_hash)
+          @captures.each do |key, value|
+            return fail("Expected line #{@line.inspect}\n    to capture #{key.inspect} as #{value.inspect} but was #{@request[key].inspect}.") if @request[key] != value
+          end
+          return true
+        else
+          return fail("The line should match the #{@line_type.inspect} line definition, but matched #{@line_hash[:line_definition].name.inspect} instead.")
+        end
+      else
+        return fail("The line did not match any line definition.")
+      end
+    end
+  end
+
+  def have_line_definition(line_type)
+    return HasLineDefinition.new(line_type)
+  end
+
+  def parse_line(line)
+    ParseLine.new(line)
+  end
+
+end

data/spec/lib/mocks.rb

@@ -0,0 +1,76 @@
+module RequestLogAnalyzer::Spec::Mocks
+
+  def mock_source
+    source = mock('RequestLogAnalyzer::Source::Base')
+    source.stub!(:file_format).and_return(testing_format)
+    source.stub!(:parsed_requests).and_return(2)
+    source.stub!(:skipped_requests).and_return(1)
+    source.stub!(:parse_lines).and_return(10)
+
+    source.stub!(:warning=)
+    source.stub!(:progress=)
+    source.stub!(:source_changes=)
+
+    source.stub!(:prepare)
+    source.stub!(:finalize)
+
+    source.stub!(:each_request).and_return do |block|
+      block.call(testing_format.request(:field => 'value1'))
+      block.call(testing_format.request(:field => 'value2'))
+    end
+
+    return source
+  end
+
+  def mock_io
+    mio = mock('IO')
+    mio.stub!(:print)
+    mio.stub!(:puts)
+    mio.stub!(:write)
+    return mio
+  end
+
+  def mock_output
+    output = mock('RequestLogAnalyzer::Output::Base')
+    output.stub!(:header)
+    output.stub!(:footer)
+    output.stub!(:puts)
+    output.stub!(:<<)
+    output.stub!(:colorize).and_return("Fancy text")
+    output.stub!(:link)
+    output.stub!(:title)
+    output.stub!(:line)
+    output.stub!(:with_style)
+    output.stub!(:table).and_yield([])
+    output.stub!(:io).and_return(mock_io)
+    output.stub!(:options).and_return({})
+    output.stub!(:slice_results).and_return { |a| a } 
+    return output
+  end
+
+  def mock_database(*stubs)
+    database = mock('RequestLogAnalyzer::Database')
+    database.stub!(:connect)
+    database.stub!(:disconnect)
+    database.stub!(:connection).and_return(mock_connection)
+    stubs.each { |s| database.stub!(s)}
+    return database
+  end
+
+  def mock_connection
+    table_creator = mock('ActiveRecord table creator')
+    table_creator.stub!(:column)
+
+    connection = mock('ActiveRecord::Base.connection')
+    connection.stub!(:add_index)
+    connection.stub!(:remove_index)
+    connection.stub!(:table_exists?).and_return(false)
+    connection.stub!(:create_table).and_yield(table_creator).and_return(true)
+    connection.stub!(:table_creator).and_return(table_creator)
+    return connection
+  end
+
+  def request_counter
+    @request_counter ||= mock('aggregator to count request')
+  end
+end

data/spec/lib/testing_format.rb

@@ -0,0 +1,46 @@
+# Simple log file specification, used to test log parser.
+class TestingFormat < RequestLogAnalyzer::FileFormat::Base
+
+  format_definition.first do |line|
+    line.header = true
+    line.teaser = /processing /
+    line.regexp = /processing request (\d+)/
+    line.captures = [{ :name => :request_no, :type => :integer }]
+  end
+
+  format_definition.test do |line|
+    line.teaser = /testing /
+    line.regexp = /testing is (\w+)(?: in (\d+\.\d+)ms)?/
+    line.captures = [{ :name => :test_capture, :type => :test_type },
+                     { :name => :duration, :type => :duration, :unit => :msec }]
+  end
+
+  format_definition.eval do |line|
+    line.regexp = /evaluation (\{.*\})/
+    line.captures = [{ :name => :evaluated, :type => :eval, :provides => { :greating => :string, :what => :string } }]
+  end
+
+  format_definition.last do |line|
+    line.footer = true
+    line.teaser = /finishing /
+    line.regexp = /finishing request (\d+)/
+    line.captures = [{ :name => :request_no, :type => :integer }]
+  end
+
+  format_definition.combined do |line|
+    line.header = true
+    line.footer = true
+    line.regexp = /this is a header and footer line/
+  end
+
+  report do |analyze|
+    analyze.frequency :test_capture, :title => 'What is testing exactly?'
+  end
+
+  class Request < RequestLogAnalyzer::Request
+    def convert_test_type(value, definition)
+      "Testing is #{value}"
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+$:.reject! { |e| e.include? 'TextMate' }
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'spec/autorun'
+require 'request_log_analyzer'
+
+module RequestLogAnalyzer::Spec
+end
+
+# Include all files in the spec_helper directory
+Dir[File.dirname(__FILE__) + "/lib/**/*.rb"].each do |file|
+  require file
+end
+
+Dir.mkdir("#{File.dirname(__FILE__)}/../tmp") unless File.exist?("#{File.dirname(__FILE__)}/../tmp")
+
+Spec::Runner.configure do |config|
+  config.include RequestLogAnalyzer::Spec::Matchers
+  config.include RequestLogAnalyzer::Spec::Mocks
+  config.include RequestLogAnalyzer::Spec::Helpers
+
+  config.extend RequestLogAnalyzer::Spec::Macros
+end

data/spec/unit/aggregator/database_inserter_spec.rb

@@ -0,0 +1,93 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Aggregator::DatabaseInserter do
+
+  before(:all) do
+    @log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format)
+  end
+
+  # The prepare method is called before the parsing starts. It should establish a connection
+  # to a database that is suitable for inserting requests later on.
+  describe '#prepare' do
+
+    before(:each) do
+      @database = mock_database(:create_database_schema!, :drop_database_schema!, :file_format=)
+      @database_inserter = RequestLogAnalyzer::Aggregator::DatabaseInserter.new(@log_parser)
+      RequestLogAnalyzer::Database.stub!(:new).and_return(@database)
+    end
+
+    it 'should establish the database connection' do
+      RequestLogAnalyzer::Database.should_receive(:new).and_return(@database)
+      @database_inserter.prepare
+    end
+
+    it "should set the file_format" do
+      @database.should_receive(:file_format=).with(testing_format)
+      @database_inserter.prepare
+    end
+
+    it 'should create the database schema during preparation' do
+      @database.should_receive(:create_database_schema!)
+      @database_inserter.prepare
+    end
+
+    it 'should not drop the database schema during preparation if not requested' do
+      @database.should_not_receive(:drop_database_schema!)
+      @database_inserter.prepare
+    end
+
+    it 'should drop the database schema during preparation if requested' do
+      @database_inserter.options[:reset_database] = true
+      @database.should_receive(:drop_database_schema!)
+      @database_inserter.prepare
+    end
+  end
+
+  test_databases.each do |name, connection|
+
+    context "using a #{name} database" do
+
+      before(:each) do
+        @database_inserter = RequestLogAnalyzer::Aggregator::DatabaseInserter.new(@log_parser, :database => connection, :reset_database => true)
+        @database_inserter.prepare
+
+        @incomplete_request = testing_format.request( {:line_type => :first, :request_no => 564})
+        @completed_request  = testing_format.request( {:line_type => :first, :request_no => 564},
+                              {:line_type => :test, :test_capture => "awesome"},
+                              {:line_type => :test, :test_capture => "indeed"},
+                              {:line_type => :eval, :evaluated => { :greating => 'howdy'}, :greating => 'howdy' },
+                              {:line_type => :last, :request_no   => 564})
+      end
+
+      after(:each) do
+        @database_inserter.database.send :remove_orm_classes!
+      end
+
+      it "should insert a record in the request table" do
+        lambda {
+          @database_inserter.aggregate(@incomplete_request)
+        }.should change(RequestLogAnalyzer::Database::Request, :count).from(0).to(1)
+      end
+
+      it "should insert a record in the first_lines table" do
+        lambda {
+          @database_inserter.aggregate(@incomplete_request)
+        }.should change(@database_inserter.database.get_class(:first), :count).from(0).to(1)
+      end
+
+      it "should insert records in all relevant line tables" do
+        @database_inserter.aggregate(@completed_request)
+        request = RequestLogAnalyzer::Database::Request.first
+        request.should have(2).test_lines
+        request.should have(1).first_lines
+        request.should have(1).eval_lines
+        request.should have(1).last_lines
+      end
+
+      it "should log a warning in the warnings table" do
+        RequestLogAnalyzer::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
+        @database_inserter.warning(:test_warning, "Testing the warning system", 12)
+      end
+    end
+  end
+end

data/spec/unit/aggregator/summarizer_spec.rb

@@ -0,0 +1,26 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Aggregator::Summarizer do
+
+  before(:each) do
+    @summarizer = RequestLogAnalyzer::Aggregator::Summarizer.new(mock_source, :output => mock_output)
+    @summarizer.prepare
+  end
+
+  it "not raise exception when creating a report after aggregating multiple requests" do
+    @summarizer.aggregate(request(:data => 'bluh1'))
+    @summarizer.aggregate(request(:data => 'bluh2'))
+
+    lambda { @summarizer.report(mock_output) }.should_not raise_error
+  end
+
+  it "not raise exception when creating a report after aggregating a single request" do
+    @summarizer.aggregate(request(:data => 'bluh1'))
+    lambda { @summarizer.report(mock_output) }.should_not raise_error
+  end
+
+  it "not raise exception when creating a report after aggregating no requests" do
+    lambda { @summarizer.report(mock_output) }.should_not raise_error
+  end
+
+end

data/spec/unit/controller/controller_spec.rb

@@ -0,0 +1,41 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Controller do
+
+  it "should use a custom output generator correctly" do
+    
+    mock_output = mock('RequestLogAnalyzer::Output::Base')
+    mock_output.stub!(:io).and_return(mock_io)
+    mock_output.should_receive(:header)
+    mock_output.should_receive(:footer)
+
+    controller  = RequestLogAnalyzer::Controller.new(mock_source, :output => mock_output)
+
+    controller.run!
+  end
+
+  it "should call aggregators correctly when run" do
+    controller  = RequestLogAnalyzer::Controller.new(mock_source, :output => mock_output)
+    
+    mock_aggregator = mock('RequestLogAnalyzer::Aggregator::Base') 
+    mock_aggregator.should_receive(:prepare).once.ordered
+    mock_aggregator.should_receive(:aggregate).with(an_instance_of(testing_format.request_class)).twice.ordered
+    mock_aggregator.should_receive(:finalize).once.ordered
+    mock_aggregator.should_receive(:report).once.ordered
+  
+    controller.aggregators << mock_aggregator
+    controller.run!
+  end
+  
+  it "should call filters when run" do
+    controller  = RequestLogAnalyzer::Controller.new(mock_source, :output => mock_output)
+    
+    mock_filter = mock('RequestLogAnalyzer::Filter::Base')
+    mock_filter.should_receive(:filter).twice.and_return(nil)
+    controller.should_receive(:aggregate_request).twice.and_return(nil)
+    
+    controller.filters << mock_filter
+    controller.run!
+  end
+  
+end

data/spec/unit/controller/log_processor_spec.rb

@@ -0,0 +1,18 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+require 'request_log_analyzer/log_processor'
+
+describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do
+
+  before(:each) do
+    @log_stripper = RequestLogAnalyzer::LogProcessor.new(testing_format, :strip, {})
+  end
+
+  it "should remove a junk line" do
+    @log_stripper.strip_line("junk line\n").should be_empty
+  end
+
+  it "should keep a teaser line intact" do
+    @log_stripper.strip_line("processing 1234\n").should be_empty
+  end
+end

data/spec/unit/database/base_class_spec.rb

@@ -0,0 +1,183 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Database::Base do
+
+  describe '.subclass_from_line_definition' do
+    before(:all) do
+      @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, { :regexp   => /Testing (\w+), tries\: (\d+)/,
+                          :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer },
+                            { :name => :evaluated, :type => :hash, :provides => {:evaluated_field => :duration} }]})
+    end
+
+    before(:each) do
+      @orm_class = mock('Line ActiveRecord::Base class')
+      @orm_class.stub!(:set_table_name)
+      @orm_class.stub!(:belongs_to)
+      @orm_class.stub!(:serialize)
+      @orm_class.stub!(:line_definition=)
+
+      Class.stub!(:new).with(RequestLogAnalyzer::Database::Base).and_return(@orm_class)
+
+      RequestLogAnalyzer::Database::Request.stub!(:has_many)
+      RequestLogAnalyzer::Database::Source.stub!(:has_many)
+
+      @database = mock_database
+      RequestLogAnalyzer::Database::Base.stub!(:database).and_return(@database)
+    end
+
+    it "should create a new subclass using the Base class as parent" do
+      Class.should_receive(:new).with(RequestLogAnalyzer::Database::Base).and_return(@orm_class)
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+    it "should store the LineDefinition" do
+      @orm_class.should_receive(:line_definition=).with(@line_definition)
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+    it "should set the table name for the subclass" do
+      @orm_class.should_receive(:set_table_name).with('test_lines')
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+    it "should set the :belongs_to relationship with the Request class" do
+      @orm_class.should_receive(:belongs_to).with(:request)
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+    it "should set a :has_many relationship in the request class" do
+      RequestLogAnalyzer::Database::Request.should_receive(:has_many).with(:test_lines)
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+    it "should set a :has_many relationship in the source class" do
+      RequestLogAnalyzer::Database::Source.should_receive(:has_many).with(:test_lines)
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+    it "should set the :belongs_to relationship with the Source class" do
+      @orm_class.should_receive(:belongs_to).with(:source)
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+    it "should serialize a complex field" do
+      @orm_class.should_receive(:serialize).with(:evaluated, Hash)
+      RequestLogAnalyzer::Database::Base.subclass_from_line_definition(@line_definition)
+    end
+
+  end
+
+  describe '.subclass_from_table' do
+    before(:each) do
+
+      RequestLogAnalyzer::Database::Request.stub!(:has_many)
+      RequestLogAnalyzer::Database::Source.stub!(:has_many)
+
+      @database = mock_database
+      @database.connection.stub!(:table_exists?).and_return(true)
+      RequestLogAnalyzer::Database::Base.stub!(:database).and_return(@database)
+
+      @klass = mock('ActiveRecord ORM class')
+      @klass.stub!(:column_names).and_return(['id', 'request_id', 'source_id', 'lineno', 'duration'])
+      @klass.stub!(:set_table_name)
+      @klass.stub!(:belongs_to)
+      Class.stub!(:new).with(RequestLogAnalyzer::Database::Base).and_return(@klass)
+    end
+
+    it "should create a new subclass using the Base class as parent" do
+      Class.should_receive(:new).with(RequestLogAnalyzer::Database::Base).and_return(@klass)
+      RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
+    end
+
+    it "should set the table name" do
+      @klass.should_receive(:set_table_name).with('completed_lines')
+      RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
+    end
+
+    it "should create the :belongs_to relation to the request class" do
+      @klass.should_receive(:belongs_to).with(:request)
+      RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
+    end
+
+    it "should create the :has_many relation in the request class" do
+      RequestLogAnalyzer::Database::Request.should_receive(:has_many).with(:completed_lines)
+      RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
+    end
+
+    it "should create the :belongs_to relation to the source class" do
+      @klass.should_receive(:belongs_to).with(:source)
+      RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
+    end
+
+    it "should create the :has_many relation in the request class" do
+      RequestLogAnalyzer::Database::Source.should_receive(:has_many).with(:completed_lines)
+      RequestLogAnalyzer::Database::Base.subclass_from_table('completed_lines')
+    end
+
+  end
+
+  describe '#create_table' do
+
+    before(:all) do
+      @line_definition = RequestLogAnalyzer::LineDefinition.new(:test, { :regexp   => /Testing (\w+), tries\: (\d+)/,
+                            :captures => [{ :name => :what, :type => :string }, { :name => :tries, :type => :integer },
+                              { :name => :evaluated, :type => :hash, :provides => {:evaluated_field => :duration} }]})
+    end
+
+    before(:each) do
+      @database = RequestLogAnalyzer::Database.new
+      @database.stub!(:connection).and_return(mock_connection)
+      @klass = @database.load_activerecord_class(@line_definition)
+      @klass.stub!(:table_exists?).and_return(false)
+    end
+
+    after(:each) do
+      @klass.drop_table!
+      @database.remove_orm_classes!
+    end
+
+    it "should call create_table with the correct table name" do
+      @database.connection.should_receive(:create_table).with(:test_lines)
+      @klass.create_table!
+    end
+
+    it "should not create a table based on the line type name if it already exists" do
+      @klass.stub!(:table_exists?).and_return(true)
+      @database.connection.should_not_receive(:create_table).with(:test_lines)
+      @klass.create_table!
+    end
+
+    it "should create an index on the request_id field" do
+      @database.connection.should_receive(:add_index).with(:test_lines, [:request_id])
+      @klass.create_table!
+    end
+
+    it "should create an index on the source_id field" do
+      @database.connection.should_receive(:add_index).with(:test_lines, [:source_id])
+      @klass.create_table!
+    end
+
+    it "should create a request_id field to link the requests together" do
+      @database.connection.table_creator.should_receive(:column).with(:request_id, :integer)
+      @klass.create_table!
+    end
+
+    it "should create a lineno field to save the location of the line in the original file" do
+      @database.connection.table_creator.should_receive(:column).with(:lineno, :integer)
+      @klass.create_table!
+    end
+
+    it "should create a field of the correct type for every defined capture field" do
+      @database.connection.table_creator.should_receive(:column).with(:what, :string)
+      @database.connection.table_creator.should_receive(:column).with(:tries, :integer)
+      @database.connection.table_creator.should_receive(:column).with(:evaluated, :text)
+      @klass.create_table!
+    end
+
+    it "should create a field of the correct type for every provided field" do
+      @database.connection.table_creator.should_receive(:column).with(:evaluated_field, :double)
+      @klass.create_table!
+    end
+  end
+end
+