request-log-analyzer 1.0.2

data/lib/request_log_analyzer/tracker/hourly_spread.rb

@@ -0,0 +1,80 @@
+module RequestLogAnalyzer::Tracker
+
+  # Determines the average hourly spread of the parsed requests.
+  # This spread is shown in a graph form.
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  #
+  # Expects the following items in the update request hash
+  # * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
+  #
+  # Example output:
+  #  Requests graph - average per day per hour
+  #   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  #    7:00 - 330 hits        : ░░░░░░░
+  #    8:00 - 704 hits        : ░░░░░░░░░░░░░░░░░
+  #    9:00 - 830 hits        : ░░░░░░░░░░░░░░░░░░░░
+  #   10:00 - 822 hits        : ░░░░░░░░░░░░░░░░░░░
+  #   11:00 - 823 hits        : ░░░░░░░░░░░░░░░░░░░
+  #   12:00 - 729 hits        : ░░░░░░░░░░░░░░░░░
+  #   13:00 - 614 hits        : ░░░░░░░░░░░░░░
+  #   14:00 - 690 hits        : ░░░░░░░░░░░░░░░░
+  #   15:00 - 492 hits        : ░░░░░░░░░░░
+  #   16:00 - 355 hits        : ░░░░░░░░
+  #   17:00 - 213 hits        : ░░░░░
+  #   18:00 - 107 hits        : ░░
+  #   ................
+  class HourlySpread < RequestLogAnalyzer::Tracker::Base
+
+    attr_reader :first, :last, :request_time_graph
+  
+    def prepare
+      options[:field] ||= :timestamp
+      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+    end
+              
+    def update(request)
+      request = request.attributes
+      timestamp = request[options[:field]]
+
+      @request_time_graph[timestamp.to_s[8..9].to_i] +=1
+      @first = timestamp if @first.nil? || timestamp < @first
+      @last  = timestamp if @last.nil?  || timestamp > @last
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+      output << "\n"
+      output << "Requests graph - average per day per hour\n"
+      output << green("━" * report_width, color) + "\n"
+      
+      if @request_time_graph == [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+        output << "None found.\n"
+        return
+      end
+
+      first_date    = DateTime.parse(@first.to_s, '%Y%m%d%H%M%S')
+      last_date     = DateTime.parse(@last.to_s, '%Y%m%d%H%M%S')
+      days          = (@last && @first) ? (last_date - first_date).ceil : 1
+      deviation     = @request_time_graph.max / 20
+      deviation     = 1 if deviation == 0
+      color_cutoff  = 15
+      
+      @request_time_graph.each_with_index do |requests, index|
+        display_chars = requests / deviation
+        request_today = requests / days
+      
+        if display_chars >= color_cutoff
+          display_chars_string = green(('░' * color_cutoff), color) + red(('░' * (display_chars - color_cutoff)), color)
+        else
+          display_chars_string = green(('░' * display_chars), color)
+        end
+      
+        output << "#{index.to_s.rjust(3)}:00 - #{(request_today.to_s + ' hits').ljust(15)} : #{display_chars_string}\n"
+      end
+      output << "\n"
+
+    end
+  end
+end

data/lib/request_log_analyzer/tracker/timespan.rb

@@ -0,0 +1,54 @@
+module RequestLogAnalyzer::Tracker
+
+  # Determines the datetime of the first request and the last request
+  # Also determines the amount of days inbetween these.
+  #
+  # Accepts the following options:
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:field</tt> The timestamp field that is looked at. Defaults to :timestamp.
+  # * <tt>:title</tt> Title do be displayed above the report.
+  #
+  # Expects the following items in the update request hash
+  # * <tt>:timestamp</tt> in YYYYMMDDHHMMSS format.
+  #
+  # Example output:
+  #  First request:        2008-07-13 06:25:06
+  #  Last request:         2008-07-20 06:18:06
+  #  Total time analyzed:  7 days
+  class Timespan < RequestLogAnalyzer::Tracker::Base
+
+    attr_reader :first, :last, :request_time_graph
+  
+    def prepare
+      options[:field] ||= :timestamp
+    end
+              
+    def update(request)
+      timestamp = request[options[:field]]
+
+      @first = timestamp if @first.nil? || timestamp < @first
+      @last  = timestamp if @last.nil?  || timestamp > @last
+    end
+  
+    def report(output = STDOUT, report_width = 80, color = false)
+      if options[:title]
+        output << "\n#{options[:title]}\n"
+        output << green('━' * options[:title].length, color) + "\n"
+      end
+      
+      first_date  = DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil
+      last_date   = DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil
+      
+      if @last && @first
+        days        = (@last && @first) ? (last_date - first_date).ceil : 1
+
+        output << "First request:        #{first_date.strftime('%Y-%m-%d %H:%M:%I')}\n"
+        output << "Last request:         #{last_date.strftime('%Y-%m-%d %H:%M:%I')}\n"
+        output << "Total time analyzed:  #{days} days\n"
+      end
+      output << "\n"
+      
+    end
+  end
+end

data/spec/controller_spec.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe RequestLogAnalyzer::Controller do
+
+  include RequestLogAnalyzerSpecHelper
+
+  # it "should include the file format module" do
+  #   controller = RequestLogAnalyzer::Controller.new(:rails)
+  #   (class << controller; self; end).ancestors.include?(RequestLogAnalyzer::FileFormat::Rails)
+  # end
+
+  it "should call the aggregators when run" do
+    file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    source      = RequestLogAnalyzer::Source::LogFile.new(file_format, :source_files => log_fixture(:rails_1x))  
+    controller  = RequestLogAnalyzer::Controller.new(source)
+    
+    mock_aggregator = mock('aggregator')
+    mock_aggregator.should_receive(:prepare).once.ordered
+    mock_aggregator.should_receive(:aggregate).with(an_instance_of(RequestLogAnalyzer::Request)).at_least(:twice).ordered
+    mock_aggregator.should_receive(:finalize).once.ordered
+    mock_aggregator.should_receive(:report).once.ordered
+    
+    another_mock_aggregator = mock('another aggregator')
+    another_mock_aggregator.should_receive(:prepare).once.ordered
+    another_mock_aggregator.should_receive(:aggregate).with(an_instance_of(RequestLogAnalyzer::Request)).at_least(:twice).ordered
+    another_mock_aggregator.should_receive(:finalize).once.ordered  
+    another_mock_aggregator.should_receive(:report).once.ordered  
+
+    
+    controller.aggregators << mock_aggregator << another_mock_aggregator
+    controller.run!
+  end
+  
+  it "should run well from the command line" do
+    temp_file = "#{File.dirname(__FILE__)}/fixtures/temp.txt"
+    system("#{File.dirname(__FILE__)}/../bin/request-log-analyzer #{log_fixture(:rails_1x)} > #{temp_file}").should be_true
+    File.unlink(temp_file)
+  end
+  
+end

data/spec/database_inserter_spec.rb

@@ -0,0 +1,101 @@
+require File.dirname(__FILE__) + '/spec_helper'
+require 'request_log_analyzer/aggregator/database'
+
+
+describe RequestLogAnalyzer::Aggregator::Database, "schema creation" do
+
+  TEST_DATABASE_FILE = File.dirname(__FILE__) + "/fixtures/requests.db"
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)
+    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
+  end
+  
+  after(:each) do
+    File.unlink(TEST_DATABASE_FILE) if File.exist?(TEST_DATABASE_FILE)
+  end
+  
+  it "should create the correct tables" do
+    ActiveRecord::Migration.should_receive(:create_table).with("warnings")    
+    ActiveRecord::Migration.should_receive(:create_table).with("requests")        
+    ActiveRecord::Migration.should_receive(:create_table).with("first_lines")
+    ActiveRecord::Migration.should_receive(:create_table).with("test_lines")        
+    ActiveRecord::Migration.should_receive(:create_table).with("last_lines")    
+    @database_inserter.prepare
+  end
+  
+  it "should create a default Request class" do
+    @database_inserter.prepare
+    SpecFormat::Database::Request.ancestors.should include(ActiveRecord::Base)
+    SpecFormat::Database::Request.column_names.should include('first_lineno')
+    SpecFormat::Database::Request.column_names.should include('last_lineno')    
+  end
+  
+  it "should create associations for the default Request class" do
+    @database_inserter.prepare
+    @request = SpecFormat::Database::Request.new
+    @request.should respond_to(:test_lines)
+    @request.test_lines.should
+  end
+  
+  it "should create the default table names" do
+    @database_inserter.prepare
+    @database_inserter.file_format.line_definitions.each do |name, definition|
+      klass = SpecFormat::Database.const_get("#{name}_line".camelize)
+      klass.column_names.should include('id')
+      klass.column_names.should include('lineno')      
+      klass.column_names.should include('request_id')      
+    end
+  end
+  
+  it "should create the correct fields in the table" do
+    @database_inserter.prepare
+    
+    SpecFormat::Database::FirstLine.column_names.should include('request_no')
+    SpecFormat::Database::LastLine.column_names.should include('request_no')
+    SpecFormat::Database::TestLine.column_names.should include('test_capture')    
+  end
+  
+end
+
+describe RequestLogAnalyzer::Aggregator::Database, "record insertion" do
+  include RequestLogAnalyzerSpecHelper  
+  
+  before(:each) do
+    log_parser = RequestLogAnalyzer::LogParser.new(spec_format)    
+    @database_inserter = RequestLogAnalyzer::Aggregator::Database.new(log_parser, :database => TEST_DATABASE_FILE)
+    @database_inserter.prepare
+        
+    @incomplete_request = RequestLogAnalyzer::Request.create(spec_format, {:line_type => :first, :request_no => 564})
+    @completed_request = RequestLogAnalyzer::Request.create(spec_format, 
+                          {:line_type => :first, :request_no  => 564},
+                          {:line_type => :test, :test_capture => "awesome"},
+                          {:line_type => :test, :test_capture => "indeed"},                                                    
+                          {:line_type => :last, :request_no   => 564})    
+  end
+  
+  after(:each) do
+    File.unlink(TEST_DATABASE_FILE) if File.exist?(TEST_DATABASE_FILE)
+  end 
+  
+  it "should insert a record in the request table" do
+    SpecFormat::Database::Request.count.should == 0
+    @database_inserter.aggregate(@incomplete_request)
+    SpecFormat::Database::Request.count.should == 1
+  end
+  
+  it "should insert records in all relevant line tables" do
+    @database_inserter.aggregate(@completed_request)
+    request = SpecFormat::Database::Request.first
+    request.should have(2).test_lines
+    request.should have(1).first_lines  
+    request.should have(1).last_lines    
+  end
+  
+  it "should log a warning in the warnings table" do
+    SpecFormat::Database::Warning.should_receive(:create!).with(hash_including(:warning_type => 'test_warning'))
+    @database_inserter.warning(:test_warning, "Testing the warning system", 12)
+  end
+  
+end

data/spec/file_format_spec.rb

@@ -0,0 +1,78 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe RequestLogAnalyzer::FileFormat, :format_definition do
+  
+  before(:each) do
+    @first_file_format = Class.new(RequestLogAnalyzer::FileFormat)
+    @second_file_format = Class.new(RequestLogAnalyzer::FileFormat)
+  end
+    
+  it "should specify lines with a hash" do
+    
+    @first_file_format.new.should have(0).line_definitions
+        
+    @first_file_format.format_definition do |line|
+      line.hash_test :regexp => /test/, :captures => []
+    end
+    
+    @format_instance = @first_file_format.new
+    @format_instance.should have(1).line_definitions
+    @format_instance.line_definitions[:hash_test].should_not be_nil
+  end
+  
+  it "should specift lines directly" do
+    @first_file_format.new.should have(0).line_definitions    
+    
+    @first_file_format.format_definition.direct_test do |line|
+      line.regexp   = /test/
+    end 
+    
+    @first_file_format.new.line_definitions[:direct_test].should_not be_nil
+  end
+ 
+  it "specify lines with a block" do
+    
+    @first_file_format.new.should have(0).line_definitions    
+    
+    @first_file_format.format_definition do |format|
+      format.block_test do |line|
+        line.regexp   = /test/
+        line.captures = []
+      end
+    end 
+    
+    @first_file_format.new.line_definitions[:block_test].should_not be_nil
+  end
+  
+  it "should define lines only for itself" do
+    
+    @first_file_format.format_definition do |line|
+      line.first_test :regexp => /test/, :captures => []
+    end
+    
+    @second_file_format.format_definition do |line|
+      line.second_test :regexp => /test/, :captures => []
+    end
+
+    @first_file_format.new.should have(1).line_definitions    
+    @first_file_format.new.line_definitions[:first_test].should_not be_nil
+    @second_file_format.new.should have(1).line_definitions        
+    @second_file_format.new.line_definitions[:second_test].should_not be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::FileFormat, :load do
+
+  include RequestLogAnalyzerSpecHelper
+
+  it "should load a predefined file format from the /file_format dir" do
+    @file_format = RequestLogAnalyzer::FileFormat.load(:rails)
+    @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails)
+  end
+  
+  it "should load a provided format file" do
+    @file_format = RequestLogAnalyzer::FileFormat.load(format_file(:spec_format))
+    @file_format.should be_kind_of(SpecFormat)
+  end
+  
+end

data/spec/file_formats/spec_format.rb

@@ -0,0 +1,26 @@
+class SpecFormat < RequestLogAnalyzer::FileFormat
+
+  format_definition.first do |line|
+    line.header = true
+    line.teaser = /processing /
+    line.regexp = /processing request (\d+)/
+    line.captures = [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]    
+  end
+  
+  format_definition.test do |line|
+    line.teaser = /testing /
+    line.regexp = /testing is (\w+)/
+    line.captures = [{ :name => :test_capture, :type => :string, :anonymize => true}]
+  end
+  
+  format_definition.last do |line|
+    line.footer = true
+    line.teaser = /finishing /
+    line.regexp = /finishing request (\d+)/
+    line.captures = [{ :name => :request_no, :type => :integer}]
+  end
+  
+  report do |analyze|
+    analyze.category :test_capture, :title => 'What is testing exactly?'
+  end
+end

data/spec/filter_spec.rb

@@ -0,0 +1,137 @@
+require File.dirname(__FILE__) + '/spec_helper'
+require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/timespan'
+require File.dirname(__FILE__) + '/../lib/request_log_analyzer/filter/field'
+
+describe RequestLogAnalyzer::Filter::Timespan, 'both before and after'  do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'), :before => DateTime.parse('2009-02-02'))
+    @filter.prepare
+  end
+  
+  it "should reject a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should be_nil
+  end
+  
+  it "should reject a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should be_nil
+  end
+  
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end
+end
+
+describe RequestLogAnalyzer::Filter::Timespan, 'only before'  do
+  include RequestLogAnalyzerSpecHelper
+  
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :before => DateTime.parse('2009-02-02'))    
+    @filter.prepare
+  end
+  
+  it "should accept a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should_not be_nil
+  end
+  
+  it "should reject a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should be_nil
+  end
+  
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end   
+end
+
+describe RequestLogAnalyzer::Filter::Timespan, 'only after'  do  
+  include RequestLogAnalyzerSpecHelper 
+  
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Timespan.new(spec_format, :after => DateTime.parse('2009-01-01'))
+    @filter.prepare
+  end
+  
+  it "should reject a request before the after date" do
+    @filter.filter(request(:timestamp => 20081212000000)).should be_nil
+  end
+  
+  it "should accept a request after the before date" do
+    @filter.filter(request(:timestamp => 20090303000000)).should_not be_nil
+  end
+  
+  it "should accept a request between the after and before dates" do
+    @filter.filter(request(:timestamp => 20090102000000)).should_not be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :select)
+    @filter.prepare
+  end
+  
+  it "should reject a request if the field value does not match" do
+    @filter.filter(request(:test => 'not test')).should be_nil
+  end
+  
+  it "should reject a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should be_nil
+  end
+
+  it "should accept a request if the both name and value match" do
+    @filter.filter(request(:test => 'test')).should_not be_nil
+  end 
+    
+  it "should accept a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should_not be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => 'test', :mode => :reject)
+    @filter.prepare
+  end
+  
+  it "should accept a request if the field value does not match" do
+    @filter.filter(request(:test => 'not test')).should_not be_nil
+  end
+  
+  it "should accept a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should_not be_nil
+  end
+
+  it "should reject a request if the both name and value match" do
+    @filter.filter(request(:test => 'test')).should be_nil
+  end 
+    
+  it "should reject a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'test'}])).should be_nil
+  end  
+end
+
+describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do
+  include RequestLogAnalyzerSpecHelper
+
+  before(:each) do
+    @filter = RequestLogAnalyzer::Filter::Field.new(spec_format, :field => :test, :value => '/test/', :mode => :select)
+    @filter.prepare
+  end
+  
+  it "should reject a request if the field value does not match" do
+    @filter.filter(request(:test => 'a working test')).should_not be_nil
+  end
+  
+  it "should reject a request if the field name does not match" do
+    @filter.filter(request(:testing => 'test')).should be_nil
+  end
+
+  it "should accept a request if the value is not the first value" do
+    @filter.filter(request([{:test => 'ignore'}, {:test => 'testing 123'}])).should_not be_nil
+  end  
+end