wvanbergen-request-log-analyzer 0.3.4 → 1.0.0

data/spec/merb_format_spec.rb

@@ -1,58 +1,58 @@
-require File.dirname(__FILE__) + '/spec_helper'
-
-describe RequestLogAnalyzer::LogParser, "Merb without combined requests" do
-  include RequestLogAnalyzerSpecHelper
-  
-  before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(:merb, :combined_requests => false)
-  end
-  
-  it "should parse a stream and find valid requests" do
-    File.open(log_fixture(:merb), 'r') do |io|
-      @log_parser.parse_io(io) do |request| 
-        request.should be_kind_of(RequestLogAnalyzer::Request)
-        request.should be_single_line
-      end
-    end
-  end
-
-  it "should find 33 request lines when lines are not linked" do
-    @log_parser.should_receive(:handle_request).exactly(33).times
-    @log_parser.parse_file(log_fixture(:merb))
-  end  
-  
-  it "should find 11 request start lines when lines are not linked" do
-    @log_parser.should_receive(:handle_request).exactly(11).times
-    @log_parser.parse_file(log_fixture(:merb), :line_types => [:started])
-  end  
-end
-
-
-describe RequestLogAnalyzer::LogParser, "Merb with combined requests" do
-  include RequestLogAnalyzerSpecHelper
-  
-  before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(:merb, :combined_requests => true)
-  end
-  
-  it "should have a valid language definitions" do
-    @log_parser.file_format.should be_valid
-  end
-  
-  it "should find 11 completed requests when lines are linked" do
-    @log_parser.should_receive(:handle_request).exactly(11).times
-    @log_parser.parse_file(log_fixture(:merb))
-  end
-  
-  it "should parse all details from a request correctly" do
-    request = nil
-    @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
-    
-    request.should be_completed
-    request[:timestamp].should == 'Fri Aug 29 11:10:23 +0200 2008'
-    request[:dispatch_time].should == 0.243424
-    request[:after_filters_time].should == 6.9e-05
-    request[:before_filters_time].should == 0.213213
-    request[:action_time].should == 0.241652
-  end
-end
+# require File.dirname(__FILE__) + '/spec_helper'
+# 
+# describe RequestLogAnalyzer::LogParser, "Merb without combined requests" do
+#   include RequestLogAnalyzerSpecHelper
+#   
+#   before(:each) do
+#     @log_parser = RequestLogAnalyzer::LogParser.new(:merb, :combined_requests => false)
+#   end
+#   
+#   it "should parse a stream and find valid requests" do
+#     File.open(log_fixture(:merb), 'r') do |io|
+#       @log_parser.parse_io(io) do |request| 
+#         request.should be_kind_of(RequestLogAnalyzer::Request)
+#         request.should be_single_line
+#       end
+#     end
+#   end
+# 
+#   it "should find 33 request lines when lines are not linked" do
+#     @log_parser.should_receive(:handle_request).exactly(33).times
+#     @log_parser.parse_file(log_fixture(:merb))
+#   end  
+#   
+#   it "should find 11 request start lines when lines are not linked" do
+#     @log_parser.should_receive(:handle_request).exactly(11).times
+#     @log_parser.parse_file(log_fixture(:merb), :line_types => [:started])
+#   end  
+# end
+# 
+# 
+# describe RequestLogAnalyzer::LogParser, "Merb with combined requests" do
+#   include RequestLogAnalyzerSpecHelper
+#   
+#   before(:each) do
+#     @log_parser = RequestLogAnalyzer::LogParser.new(:merb, :combined_requests => true)
+#   end
+#   
+#   it "should have a valid language definitions" do
+#     @log_parser.file_format.should be_valid
+#   end
+#   
+#   it "should find 11 completed requests when lines are linked" do
+#     @log_parser.should_receive(:handle_request).exactly(11).times
+#     @log_parser.parse_file(log_fixture(:merb))
+#   end
+#   
+#   it "should parse all details from a request correctly" do
+#     request = nil
+#     @log_parser.parse_file(log_fixture(:merb)) { |found_request| request ||= found_request }
+#     
+#     request.should be_completed
+#     request[:timestamp].should == DateTime.parse('Fri Aug 29 11:10:23 +0200 2008')
+#     request[:dispatch_time].should == 0.243424
+#     request[:after_filters_time].should == 6.9e-05
+#     request[:before_filters_time].should == 0.213213
+#     request[:action_time].should == 0.241652
+#   end
+# end

data/spec/rails_format_spec.rb

@@ -4,7 +4,7 @@ describe RequestLogAnalyzer::LogParser, "Rails without combined requests" do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(:rails, :combined_requests => false)
+    @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails), :combined_requests => false)
   end
   
   it "should parse a stream and find valid requests" do
@@ -20,7 +20,7 @@ describe RequestLogAnalyzer::LogParser, "Rails without combined requests" do
     @log_parser.parse_file(log_fixture(:rails_1x)) { |request| requests << request }
     requests.length.should == 8
     requests.each { |r| r.should be_single_line }
-    requests.select { |r| r.line_type == :started }.should have(4).items
+    requests.select { |r| r.line_type == :processing }.should have(4).items
   end  
 end
 
@@ -29,7 +29,7 @@ describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
   include RequestLogAnalyzerSpecHelper
   
   before(:each) do
-    @log_parser = RequestLogAnalyzer::LogParser.new(:rails, :combined_requests => true)
+    @log_parser = RequestLogAnalyzer::LogParser.new(RequestLogAnalyzer::FileFormat.load(:rails), :combined_requests => true)
   end
   
   it "should have a valid language definitions" do
@@ -45,7 +45,7 @@ describe RequestLogAnalyzer::LogParser, "Rails with combined requests" do
   it "should parse a Rails 2.2 request properly" do
     @log_parser.should_not_receive(:warn)
     @log_parser.parse_file(log_fixture(:rails_22)) do |request|
-      request.should =~ :started
+      request.should =~ :processing
       request.should =~ :completed  
     
       request[:controller].should == 'PageController'

data/spec/request_spec.rb

@@ -2,8 +2,10 @@ require File.dirname(__FILE__) + '/spec_helper'
 
 describe RequestLogAnalyzer::Request, :single_line do
   
+  include RequestLogAnalyzerSpecHelper
+  
   before(:each) do
-    @single_line_request = RequestLogAnalyzer::Request.new(TestFileFormat)
+    @single_line_request = RequestLogAnalyzer::Request.new(spec_format)
     @single_line_request << { :line_type => :test, :lineno => 1, :test_capture => 'awesome!' }
   end
   
@@ -38,8 +40,10 @@ end
 
 describe RequestLogAnalyzer::Request, :combined do
   
+  include RequestLogAnalyzerSpecHelper
+  
   before(:each) do
-    @combined_request = RequestLogAnalyzer::Request.new(TestFileFormat)
+    @combined_request = RequestLogAnalyzer::Request.new(spec_format)
     @combined_request << { :line_type => :first, :lineno =>  1, :name => 'first line!' }    
     @combined_request << { :line_type => :test,  :lineno =>  4, :test_capture => 'testing' }        
     @combined_request << { :line_type => :test,  :lineno =>  7, :test_capture => 'testing some more' }            

data/spec/spec_helper.rb

@@ -2,15 +2,33 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'rubygems'
 require 'spec'
-
 require 'request_log_analyzer'
 
 module RequestLogAnalyzerSpecHelper
   
+  def format_file(format)
+    File.dirname(__FILE__) + "/file_formats/#{format}.rb"
+  end
+  
+  def spec_format
+    @spec_format ||= begin
+      require format_file(:spec_format)
+      SpecFormat.new
+    end
+  end
+      
   def log_fixture(name)
     File.dirname(__FILE__) + "/fixtures/#{name}.log"
   end
   
+  def request(fields, format = TestFileFormat)
+    if fields.kind_of?(Array)
+      RequestLogAnalyzer::Request.create(format, *fields)
+    else
+      RequestLogAnalyzer::Request.create(format, fields)
+    end
+  end
+  
 end
 
 module TestFileFormat
@@ -32,18 +50,18 @@ module TestFileFormat
       :header => true,
       :teaser => /processing /,
       :regexp => /processing request (\d+)/,
-      :captures => [{:request_no => :integer}]    
+      :captures => [{ :name => :request_no, :type => :integer, :anonymize => :slightly }]    
     },
     :test => {
       :teaser => /testing /,
       :regexp => /testing is (\w+)/,
-      :captures => [{:test_capture => :string}]
+      :captures => [{ :name => :test_capture, :type => :string, :anonymize => true}]
     }, 
     :last => {
       :footer => true,
       :teaser => /finishing /,
       :regexp => /finishing request (\d+)/,
-      :captures => [{:request_no => :integer}]
+      :captures => [{ :name => :request_no, :type => :integer}]
     }
   }
 end

data/spec/summarizer_spec.rb

@@ -4,106 +4,12 @@ require 'request_log_analyzer/aggregator/summarizer'
 describe RequestLogAnalyzer::Aggregator::Summarizer, :single_line do
   
   before(:each) do
-    @summarizer =   RequestLogAnalyzer::Aggregator::Summarizer.new(TestFileFormat, :combined_requests => false)
-    @summarizer.prepare
-    
-    @test_request_1 = RequestLogAnalyzer::Request.create(TestFileFormat, {:line_type => :first, :request_no => 564})
-    @test_request_2 = RequestLogAnalyzer::Request.create(TestFileFormat, {:line_type => :test, :test_capture => 'awesome'})
-    @test_request_3 = RequestLogAnalyzer::Request.create(TestFileFormat, {:line_type => :last, :request_no => 564})        
   end
-  
-  it "should include the file format Summarizer module" do
-    metaclass = (class << @summarizer; self; end)
-    metaclass.ancestors.should include(TestFileFormat::Summarizer)
-    @summarizer.class.ancestors.should_not include(TestFileFormat::Summarizer)
-  end
-  
-  it "should set the default bucket for a new line type" do
-    @summarizer.should_receive(:default_bucket_content).once.and_return(:count => 0)
-    @summarizer.aggregate(@test_request_1)
-  end
-  
-  it "should request a bucket name for the hash" do
-    @summarizer.should_receive(:bucket_for).with(@test_request_1).once.and_return('all')
-    @summarizer.aggregate(@test_request_1)
-  end  
-
-  it "should register" do
-    @summarizer.should_receive(:update_bucket).with(anything, @test_request_1).once
-    @summarizer.aggregate(@test_request_1)
-  end
-
-  it "should have buckets for every line type on the first level" do
-    @summarizer.aggregate(@test_request_1)
-    @summarizer.aggregate(@test_request_2)    
-    @summarizer.aggregate(@test_request_3)    
-    @summarizer.buckets.should have(3).items
-    @summarizer.buckets.should have_key(:first)
-    @summarizer.buckets.should have_key(:test)
-    @summarizer.buckets.should have_key(:last)    
-  end
-  
-  it "should aggregate in the same bucket" do
-    @summarizer.should_receive(:bucket_for).exactly(3).times.and_return('all')
-    3.times { @summarizer.aggregate(@test_request_2) }
-    @summarizer.buckets[:test].should have(1).items
-  end
-
-  it "should aggregate in different buckets based on" do
-    4.times do |n|
-      @summarizer.stub!(:bucket_for).and_return("bucket #{n % 2}") # buckets 1 and 2
-      @summarizer.aggregate(@test_request_2)
-    end
-    @summarizer.buckets[:test].should have(2).items
-  end
-  
 end
 
 describe RequestLogAnalyzer::Aggregator::Summarizer, :combined_requests do
   
   before(:each) do
-    @summarizer = RequestLogAnalyzer::Aggregator::Summarizer.new(TestFileFormat, :combined_requests => true)
-    @summarizer.prepare
-    
-    @test_request = RequestLogAnalyzer::Request.create(TestFileFormat, 
-          {:line_type => :first, :request_no => 564},
-          {:line_type => :test, :test_capture => 'blug'},
-          {:line_type => :last, :request_no => 564})
-  end
-  
-  it "should include the file format Summarizer module" do
-    metaclass = (class << @summarizer; self; end)
-    metaclass.ancestors.should include(TestFileFormat::Summarizer)
-    @summarizer.class.ancestors.should_not include(TestFileFormat::Summarizer)
-  end
-  
-  it "should set the default bucket for a new line type" do
-    @summarizer.should_receive(:default_bucket_content).once.and_return(:count => 0)
-    @summarizer.aggregate(@test_request)
   end
 
-  it "should register" do
-    @summarizer.should_receive(:update_bucket).with(anything, @test_request).once
-    @summarizer.aggregate(@test_request)
-  end
-  
-  it "should aggregate in the same bucket" do
-    3.times { @summarizer.aggregate(@test_request) }
-    @summarizer.buckets.should have(1).items
-  end
-
-  it "should aggregate in the same bucket" do
-    @summarizer.should_receive(:bucket_for).exactly(3).times.and_return('all')
-    3.times { @summarizer.aggregate(@test_request) }
-    @summarizer.buckets.should have(1).items
-  end
-
-  it "should aggregate in different buckets based on" do
-    4.times do |n|
-      @summarizer.stub!(:bucket_for).and_return("bucket #{n % 2}") # buckets 1 and 2
-      @summarizer.aggregate(@test_request)
-    end
-    @summarizer.buckets.should have(2).items
-  end
-  
 end

data/tasks/github-gem.rake

@@ -135,7 +135,7 @@ module Rake
       spec = File.read(gemspec_file)
       spec.gsub! /^(\s* s.(test_)?files \s* = \s* )( \[ [^\]]* \] | %w\( [^)]* \) )/mx do
         assignment = $1
-        bunch = $2 ? list.grep(/^test.*_test\.rb$/) : list
+        bunch = $2 ? list.grep(/^(test.*_test\.rb|spec.*_spec.rb)$/) : list
         '%s%%w(%s)' % [assignment, bunch.join(' ')]
       end
 

data/test/rails_log_parser_test.rb

@@ -91,9 +91,4 @@ class RailsLogParserTest < Test::Unit::TestCase
   
   end
   
-  def test_virtual_mongrel
-    request_log_analyzer = RequestLogAnalyzer.new()
-    request_log_analyzer.analyze_with_virtual_mongrels([fragment_file(5)])
-  end
-  
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wvanbergen-request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 0.3.4
+  version: 1.0.0
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-12-29 00:00:00 -08:00
+date: 2009-10-01 00:00:00 -07:00
 default_executable: request-log-analyzer
 dependencies: []
 

data/lib/base/summarizer.rb

@@ -1,85 +0,0 @@
-module Base
-  
-  # Functions to summarize an array of requets.
-  # Can calculate request counts, duratations, mean times etc. of all the requests given.
-  class Summarizer
-    attr_reader :actions
-    attr_reader :errors
-    attr_reader :request_count
-    attr_reader :request_time_graph
-    attr_reader :first_request_at
-    attr_reader :last_request_at
-    attr_reader :methods
-
-    attr_accessor :blocker_duration
-    DEFAULT_BLOCKER_DURATION = 1.0
-   
-    # Initializer. Sets global variables
-    # Options
-    def initialize(options = {})
-      @actions  = {}
-      @blockers = {}
-      @errors   = {}
-      @request_count = 0
-      @blocker_duration = DEFAULT_BLOCKER_DURATION
-      @request_time_graph = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-      @methods = {:GET => 0, :POST => 0, :PUT => 0, :DELETE => 0, :unknown => 0}
-      
-      self.initialize_hook(options) if self.respond_to?(:initialize_hook)
-    end
-   
-    # Check if any of the request parsed had a timestamp.
-    def has_timestamps?
-      @first_request_at
-    end
-   
-    # Calculate the duration of a request
-    # Returns a DateTime object if possible, 0 otherwise.
-    def duration
-      (@last_request_at && @first_request_at) ? (DateTime.parse(@last_request_at) - DateTime.parse(@first_request_at)).ceil : 0
-    end
-    
-    # Check if the request time graph usable data.
-    def request_time_graph?
-      @request_time_graph.uniq != [0] && duration > 0
-    end
-
-    # Return a list of requests sorted on a specific action field
-    # <tt>field</tt> The action field to sort by.
-    # <tt>min_count</tt> Values which fall below this amount are not returned (default nil).
-    def sort_actions_by(field, min_count = nil)
-      actions = min_count.nil? ? @actions.to_a : @actions.delete_if { |k, v| v[:count] < min_count}.to_a
-      actions.sort { |a, b| (a[1][field.to_sym] <=> b[1][field.to_sym]) }
-    end
-
-    # Returns a list of request blockers sorted by a specific field
-    # <tt>field</tt> The action field to sort by.
-    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
-    def sort_blockers_by(field, min_count = @blocker_duration)
-      blockers = min_count.nil? ? @blockers.to_a : @blockers.delete_if { |k, v| v[:count] < min_count}.to_a
-      blockers.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
-    end
-
-    # Returns a list of request blockers sorted by a specific field
-    # <tt>field</tt> The action field to sort by.
-    # <tt>min_count</tt> Values which fall below this amount are not returned (default @blocker_duration).
-    def sort_errors_by(field, min_count = nil)
-      errors = min_count.nil? ? @errors.to_a : @errors.delete_if { |k, v| v[:count] < min_count}.to_a
-      errors.sort { |a, b| a[1][field.to_sym] <=> b[1][field.to_sym] } 
-    end
-    
-    # Compare date strings fast
-    # Assumes date formats: "2008-07-14 12:11:20" or alike.
-    # <tt>first_date</tt> The first date string
-    # <tt>second_date</tt> The second date string
-    # Returns -1 if first_date < second_date, nil if equal
-    # and 1 if first_date > second_date (<=>)
-    def hamburger_compare_string_dates first_date, second_date
-      return nil if first_date.nil? || second_date.nil?
-      
-      first_date.gsub(/[^0-9|\s]/,'').to_i \
-        <=> \
-      second_date.gsub(/[^0-9|\s]/,'').to_i
-    end
-  end
-end 

data/lib/bashcolorizer.rb

@@ -1,60 +0,0 @@
-# Colorize a text output with the given color if.
-# <tt>text</tt> The text to colorize.
-# <tt>color_code</tt> The color code string to set
-# <tt>color</tt> Does not color if false. Defaults to ($arguments && $arguments[:colorize])
-def colorize(text, color_code, color = $colorize)
-  color ? "#{color_code}#{text}\e[0m" : text
-end
-
-# Draw a red line of text
-def red(text)
-  colorize(text, "\e[31m")
-end
-
-# Draw a Green line of text
-def green(text)
-  colorize(text, "\e[32m")
-end
-
-# Draw a Yellow line of text
-def yellow(text)
-  colorize(text, "\e[33m")
-end
-
-# Draw a Yellow line of text
-def blue(text)
-  colorize(text, "\e[34m")
-end
-
-def white(text)
-  colorize(text, "\e[37m")
-end
-
-
-#STYLE = {
-#      :default    =>    “33[0m”,
-#       # styles
-#       :bold       =>    “33[1m”,
-#       :underline  =>    “33[4m”,
-#       :blink      =>    “33[5m”,
-#       :reverse    =>    “33[7m”,
-#       :concealed  =>    “33[8m”,
-#      # font colors
-#       :black      =>    “33[30m”,
-#       :red        =>    “33[31m”,
-#       :green      =>    “33[32m”,
-#       :yellow     =>    “33[33m”,
-#       :blue       =>    “33[34m”,
-#       :magenta    =>    “33[35m”,
-#       :cyan       =>    “33[36m”,
-#       :white      =>    “33[37m”,
-#       # background colors
-#       :on_black   =>    “33[40m”,
-#       :on_red     =>    “33[41m”,
-#       :on_green   =>    “33[42m”,
-#       :on_yellow  =>    “33[43m”,
-#       :on_blue    =>    “33[44m”,
-#       :on_magenta =>    “33[45m”,
-#       :on_cyan    =>    “33[46m”,
-#       :on_white   =>    “33[47m” }
-#

data/lib/command_line/exceptions.rb

@@ -1,37 +0,0 @@
-module CommandLine
-  
-  # Commandline parsing errors and exceptions
-  class Error < Exception
-  end
-
-  # Missing a required flag
-  class FlagMissing < CommandLine::Error
-  end
-
-  # Missing a required file
-  class FileMissing < CommandLine::Error
-  end
-
-  # Missing a required flag argument
-  class FlagExpectsArgument < CommandLine::Error
-    def initialize(flag)
-      super("#{flag} expects an argument!")
-    end    
-  end
-
-  # Missing a required command
-  class CommandMissing < CommandLine::Error
-    def initialize(msg = "A command is missing")
-      super(msg)
-    end
-    
-  end
-
-  # Encountered an unkown flag
-  class UnknownFlag < CommandLine::Error
-    def initialize(flag)
-      super("#{flag} not recognized as a valid flag!")
-    end
-  end
-  
-end

data/lib/merb_analyzer/summarizer.rb

@@ -1,61 +0,0 @@
-require 'date'
-module MerbAnalyzer
-  
-  # Functions to summarize an array of requets.
-  # Can calculate request counts, duratations, mean times etc. of all the requests given.
-  class Summarizer < Base::Summarizer
-
-    # Initializer. Sets global variables
-    def initialize_hook(options = {})
-      @hash_cache = nil
-    end
-
-    # Parse a request string into a hash containing all keys found in the string.
-    # Yields the hash found to the block operator.
-    # <tt>request</tt> The request string to parse.
-    # <tt>&block</tt> Block operator
-    def group(request, &block)
-      request[:duration] ||= 0
-      
-      case request[:type]   
-        when :started 
-          @first_request_at ||= request[:timestamp] # assume time-based order of file
-          @last_request_at  = request[:timestamp]   # assume time-based order of file
-          @request_time_graph[request[:timestamp][11..12].to_i] +=1
-          
-        when :params
-          params_hash = {}
-          request[:raw_hash].split(',').collect{|x| x.split('=>')}.each do |k,v|
-            key = k.gsub('"', '').gsub(' ', '').to_sym
-            value = v.gsub('"', '')
-            request.store(key, value)
-          end
-          
-          hash = block_given? ? yield(request) : request.hash
-
-          @actions[hash] ||= {:count => 0, :total_time => 0.0, :total_db_time => 0.0, :total_rendering_time => 0.0, 
-                                 :min_time => request[:duration], :max_time => request[:duration] }
-
-          @actions[hash][:count] += 1
-          request[:method] = 'GET' unless request[:method]
-          @methods[request[:method].upcase.to_sym] += 1
-
-          @hash_cache = hash
-        when :completed
-          @request_count += 1 
-          
-          @actions[@hash_cache][:total_time] += request[:dispatch_time]
-          @actions[@hash_cache][:mean_time] = @actions[@hash_cache][:total_time] / @actions[@hash_cache][:count].to_f          
-          @actions[@hash_cache][:min_time] = [@actions[@hash_cache][:min_time], request[:dispatch_time]].min
-          @actions[@hash_cache][:max_time] = [@actions[@hash_cache][:min_time], request[:dispatch_time]].max
-          
-          @actions[@hash_cache][:total_db_time] = 0
-          @actions[@hash_cache][:mean_db_time] = 0
-          @actions[@hash_cache][:mean_rendering_time] = 0
-          
-        when :failed
-
-      end
-    end
-  end
-end