wvanbergen-request-log-analyzer 0.3.3 → 0.3.4

data/test/merb_log_parser_test.rb

@@ -0,0 +1,39 @@
+require 'test/unit'
+
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/merb_analyzer/log_parser"
+
+class MerbLogParserTest < Test::Unit::TestCase
+  
+  def fragment_file(number)
+    "#{File.dirname(__FILE__)}/log_fragments/merb_#{number}.log"
+  end
+  
+  def test_parse_started_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:started) do |request|
+      requests << request
+    end
+    assert_equal requests[0][:timestamp], "Fri Aug 29 11:10:23 +0200 2008"
+  end
+
+  def test_parse_completed_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:completed) do |request|
+      requests << request
+    end
+    
+    assert_equal requests[0][:action_time], 0.241652
+  end
+
+  def test_parse_params_merb_fragment
+    requests = []
+    parser = MerbAnalyzer::LogParser.new(fragment_file(1)).each(:params) do |request|
+      requests << request
+    end
+    
+    assert_match '"controller"=>"session"', requests[0][:raw_hash]
+    assert_match '"action"=>"destroy"', requests[0][:raw_hash]
+  end
+  
+end

data/test/rails_log_parser_test.rb

@@ -0,0 +1,99 @@
+require 'test/unit'
+
+require "#{File.dirname(__FILE__)}/../lib/request_log_analyzer"
+
+class RailsLogParserTest < Test::Unit::TestCase
+  
+  def fragment_file(number)
+    "#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
+  end
+  
+  
+  def test_rails_22_log_format
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(3)).each(:completed) do |request|
+      count += 1
+      assert_equal 0.614, request[:duration]  # should be 0.614
+    end
+    assert_equal 1, count  
+  end
+  
+  def test_progress_messages
+    log_file = fragment_file(1)
+  
+    finished_encountered = false
+    file_size = File.size(log_file)
+    
+    previous_pos = -1
+    parser = RailsAnalyzer::LogParser.new(log_file)
+    parser.progress do |pos, total|
+      assert_equal file_size, total
+      if pos == :finished
+        finished_encountered = true 
+      else
+        assert pos <= total
+        assert pos > previous_pos
+        previous_pos = pos
+      end
+    end
+    
+    # now parse the file
+    parser.each(:started) { }
+    
+    assert finished_encountered, "A finished event should have been fired"
+  end
+  
+  def test_parse_mongrel_log_fragment
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started) { count += 1 }
+    assert_equal 4, count
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:completed) { count += 1 }
+    assert_equal 4, count    
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(1)).each(:started, :completed) { count += 1 }
+    assert_equal 8, count    
+  end
+  
+  def test_parse_syslog_fragment
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) { count += 1 }
+    assert_equal 1, count
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) { count += 1 }
+    assert_equal 1, count    
+    
+    count = 0
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started, :completed) { count += 1 }
+    assert_equal 2, count    
+  end
+  
+  def test_parse_syslog_fragment_content
+    # this test only works because there is only one requests in the fragment
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:started) do |request|
+      assert_equal "EmployeeController", request[:controller]
+      assert_equal "index", request[:action]      
+      assert_equal "GET", request[:method]
+      assert_equal '10.1.1.33', request[:ip]
+      assert_equal '2008-07-13 06:25:58', request[:timestamp]            
+    end
+  
+    parser = RailsAnalyzer::LogParser.new(fragment_file(2)).each(:completed) do |request|
+      assert_equal "http://example.com/employee.xml", request[:url]
+      assert_equal 200, request[:status]
+      assert_equal 0.21665, request[:duration]
+      assert_equal 0.00926, request[:rendering]      
+      assert_equal 0.0, request[:db]
+    end
+  
+  end
+  
+  def test_virtual_mongrel
+    request_log_analyzer = RequestLogAnalyzer.new()
+    request_log_analyzer.analyze_with_virtual_mongrels([fragment_file(5)])
+  end
+  
+end

data/test/record_inserter_test.rb

@@ -0,0 +1,45 @@
+require 'test/unit'
+
+require "#{File.dirname(__FILE__)}/../lib/base/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/base/record_inserter"
+
+require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/log_parser"
+require "#{File.dirname(__FILE__)}/../lib/rails_analyzer/record_inserter"
+
+class RecordInserterTest < Test::Unit::TestCase
+  
+  def fragment_file(number)
+    "#{File.dirname(__FILE__)}/log_fragments/fragment_#{number}.log"
+  end
+  
+  def setup
+    File.delete('_tmp.db') if File.exist?('_tmp.db')
+  end
+  
+  def teardown
+    File.delete('_tmp.db') if File.exist?('_tmp.db')
+  end
+  
+  def test_insert_log_fragment
+    
+    db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
+      RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
+    end
+
+    assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
+    assert_equal 4, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i 
+  end
+  
+  def test_insert_multiple_fragments
+    RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
+      RailsAnalyzer::LogParser.new(fragment_file(1)).each { |request| batch.insert(request) }
+    end
+
+    db = RailsAnalyzer::RecordInserter.insert_batch_into('_tmp.db') do |batch|
+      RailsAnalyzer::LogParser.new(fragment_file(2)).each { |request| batch.insert(request) }
+    end
+    assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM started_requests").to_i
+    assert_equal 5, db.database.get_first_value("SELECT COUNT(*) FROM completed_requests").to_i    
+  end
+  
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-12-28 00:00:00 -08:00
+date: 2008-12-29 00:00:00 -08:00
 default_executable: request-log-analyzer
 dependencies: []
 
@@ -18,6 +18,7 @@ description: Rails log analyzer's purpose is to find what actions are best candi
 email: willem@vanbergen.org
 executables: 
 - request-log-analyzer
+- request-log-database
 extensions: []
 
 extra_rdoc_files: []
@@ -29,8 +30,11 @@ files:
 - TODO
 - bin
 - bin/request-log-analyzer
+- bin/request-log-database
 - lib
 - lib/base
+- lib/base/log_parser.rb
+- lib/base/record_inserter.rb
 - lib/base/summarizer.rb
 - lib/bashcolorizer.rb
 - lib/command_line
@@ -38,24 +42,14 @@ files:
 - lib/command_line/exceptions.rb
 - lib/command_line/flag.rb
 - lib/merb_analyzer
+- lib/merb_analyzer/log_parser.rb
 - lib/merb_analyzer/summarizer.rb
 - lib/rails_analyzer
+- lib/rails_analyzer/log_parser.rb
+- lib/rails_analyzer/record_inserter.rb
 - lib/rails_analyzer/summarizer.rb
 - lib/rails_analyzer/virtual_mongrel.rb
-- lib/request_log_analyzer
 - lib/request_log_analyzer.rb
-- lib/request_log_analyzer/aggregator
-- lib/request_log_analyzer/aggregator/base.rb
-- lib/request_log_analyzer/aggregator/database.rb
-- lib/request_log_analyzer/aggregator/echo.rb
-- lib/request_log_analyzer/aggregator/summarizer.rb
-- lib/request_log_analyzer/controller.rb
-- lib/request_log_analyzer/file_format
-- lib/request_log_analyzer/file_format.rb
-- lib/request_log_analyzer/file_format/merb.rb
-- lib/request_log_analyzer/file_format/rails.rb
-- lib/request_log_analyzer/log_parser.rb
-- lib/request_log_analyzer/request.rb
 - lib/ruby-progressbar
 - lib/ruby-progressbar/progressbar.en.rd
 - lib/ruby-progressbar/progressbar.ja.rd
@@ -97,9 +91,19 @@ files:
 - tasks
 - tasks/github-gem.rake
 - tasks/request_log_analyzer.rake
-- tasks/rspec.rake
+- tasks/test.rake
 - test
 - test/base_summarizer_test.rb
+- test/log_fragments
+- test/log_fragments/fragment_1.log
+- test/log_fragments/fragment_2.log
+- test/log_fragments/fragment_3.log
+- test/log_fragments/fragment_4.log
+- test/log_fragments/fragment_5.log
+- test/log_fragments/merb_1.log
+- test/merb_log_parser_test.rb
+- test/rails_log_parser_test.rb
+- test/record_inserter_test.rb
 has_rdoc: false
 homepage: http://github.com/wvanbergen/request-log-analyzer/wikis
 post_install_message: 
@@ -128,3 +132,6 @@ specification_version: 2
 summary: A command line tool to analyze Rails logs
 test_files: 
 - test/base_summarizer_test.rb
+- test/merb_log_parser_test.rb
+- test/rails_log_parser_test.rb
+- test/record_inserter_test.rb

data/lib/request_log_analyzer/aggregator/base.rb

@@ -1,34 +0,0 @@
-module RequestLogAnalyzer::Aggregator
-
-
-  class Base
-    
-    include RequestLogAnalyzer::FileFormat
-    
-    attr_reader :options
-    
-    def initialize(format, options = {})
-      self.register_file_format(format)
-      @options = options
-    end
-    
-    def aggregate(request)
-      # implement me!
-    end
-    
-    
-    def prepare
-    end
-    
-    def finalize
-    end
-    
-    def warning(type, message, lineno)
-    end    
-
-    def report(color = false)
-      
-    end
-
-  end
-end

data/lib/request_log_analyzer/aggregator/database.rb

@@ -1,86 +0,0 @@
-require 'rubygems'
-require 'activerecord'
-
-module RequestLogAnalyzer::Aggregator
-
-  class Database < Base
-
-    attr_reader :request_id
-
-    def prepare
-      ActiveRecord::Base.establish_connection(:adapter => 'sqlite3', :database => options[:database])
-
-      File.unlink(options[:database]) if File.exist?(options[:database])
-      create_database_schema!
-      
-      @request_id = 0
-    end
-    
-    def aggregate(request)
-      @request_id += 1
-      
-      request.lines.each do |line|
-        class_name = "#{line[:line_type]}_line".camelize #split(/[^a-z0-9]/i).map{ |w| w.capitalize }.join('')
-        
-        attributes = line.reject { |k, v| [:line_type].include?(k) }
-        attributes[:request_id] = @request_id if options[:combined_requests]  
-        file_format.const_get(class_name).create!(attributes)
-      end
-      
-    end
-    
-    def warning(type, message, lineno)
-      file_format::Warning.create!(:warning_type => type.to_s, :message => message, :lineno => lineno)
-    end
-    
-    protected 
-    
-    def create_database_table(name, definition)
-      ActiveRecord::Migration.suppress_messages do
-        ActiveRecord::Migration.create_table("#{name}_lines") do |t|
-          t.column(:request_id, :integer) #if options[:combined_requests]
-          t.column(:lineno, :integer)
-          definition.captures.each do |field|
-            # there is only on key/value pait in this hash
-            field.each { |key, capture_type| t.column(key, column_type(capture_type)) }
-          end
-        end
-      end
-    end
-
-    def create_warning_table_and_class
-      ActiveRecord::Migration.suppress_messages do
-        ActiveRecord::Migration.create_table("warnings") do |t|
-          t.string  :warning_type, :limit => 30, :null => false
-          t.string  :message
-          t.integer :lineno          
-        end
-      end    
-      
-      file_format.const_set('Warning', Class.new(ActiveRecord::Base)) unless file_format.const_defined?('Warning')
-    end
-
-    def create_activerecord_class(name, definition)
-      class_name = "#{name}_line".camelize
-      file_format.const_set(class_name, Class.new(ActiveRecord::Base)) unless file_format.const_defined?(class_name)
-    end
-
-    def create_database_schema!
-      file_format.line_definitions.each do |name, definition|
-        create_database_table(name, definition)
-        create_activerecord_class(name, definition)
-      end
-      
-      create_warning_table_and_class
-    end
-    
-    def column_type(capture_type)
-      case capture_type
-      when :sec;   :double
-      when :msec;  :double
-      when :float; :double
-      else capture_type
-      end
-    end
-  end
-end

data/lib/request_log_analyzer/aggregator/echo.rb

@@ -1,10 +0,0 @@
-module RequestLogAnalyzer::Aggregator
-
-  class Echo < Base
-    
-    def aggregate(request)
-      puts "\nRequest: " + request.inspect
-    end
-
-  end
-end

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -1,53 +0,0 @@
-module RequestLogAnalyzer::Aggregator
-
-  class Summarizer < Base
-    
-    attr_reader :buckets
-    
-    def prepare
-      @buckets = {}
-    end
-    
-    def aggregate(request)
-      if options[:combined_requests]
-        current_bucket_hash = @buckets
-      else       
-        @buckets[request.line_type] ||= {}
-        current_bucket_hash = @buckets[request.line_type]
-      end
-      
-      bucket_name = bucket_for(request)
-      current_bucket_hash[bucket_name] ||= default_bucket_content
-      update_bucket(current_bucket_hash[bucket_name], request)
-    end
-    
-    def default_bucket_content
-      return { :count => 0 }
-    end
-    
-    def update_bucket(bucket, request)
-      bucket[:count] += 1
-    end    
-    
-    def bucket_for(request)
-      'all'
-    end
-       
-    def report(color = false)
-      if options[:combined_requests]
-        @buckets.each do |hash, values|
-          puts "  #{hash[0..40].ljust(41)}: #{values[:count]}"
-        end     
-      else
-        @buckets.each do |line_type, buckets|
-          puts "Line type #{line_type.inspect}:"
-          buckets.each do |hash, values|
-            puts "  #{hash[0..40].ljust(41)}: #{values[:count]}"
-          end
-        end
-      end
-    end
-    
-
-  end
-end