request-log-analyzer 1.3.4 → 1.3.5

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -57,11 +57,14 @@ module RequestLogAnalyzer::Aggregator
     
     # Records source changes in the sources table
     def source_change(change, filename)
-      case change
-      when :started
-        @sources[filename] = database.source_class.create!(:filename => filename)
-      when :finished
-        @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
+      if File.exist?(filename)
+        case change
+        when :started
+          p database.source_class
+          @sources[filename] = database.source_class.create!(:filename => filename)
+        when :finished
+          @sources[filename].update_attributes!(:filesize => File.size(filename), :mtime => File.mtime(filename))
+        end
       end
     end
     

data/lib/request_log_analyzer/database/base.rb

@@ -13,8 +13,9 @@ class RequestLogAnalyzer::Database::Base < ActiveRecord::Base
   def line_type
     self.class.name.underscore.gsub(/_line$/, '').to_sym
   end
-
-  cattr_accessor :database, :line_definition
+  
+  class_inheritable_accessor :line_definition
+  cattr_accessor :database
 
   def self.subclass_from_line_definition(definition)
     klass = Class.new(RequestLogAnalyzer::Database::Base)

data/lib/request_log_analyzer/file_format/amazon_s3.rb

@@ -9,7 +9,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :access do |line|
       line.header = true
       line.footer = true
-      line.regexp = /^([^\ ]+) ([^\ ]+) \[([^\]]{26})\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
+      line.regexp = /^([^\ ]+) ([^\ ]+) \[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\] (\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) (\d+) (\d+) (\d+) (\d+) "([^"]+)" "([^"]+)"/
       line.captures << { :name => :bucket_owner,    :type => :string } << 
                        { :name => :bucket,          :type => :string } << 
                        { :name => :timestamp,       :type => :timestamp } <<
@@ -21,8 +21,8 @@ module RequestLogAnalyzer::FileFormat
                        { :name => :request_uri,     :type => :string } <<
                        { :name => :http_status,     :type => :integer } <<
                        { :name => :error_code,      :type => :nillable_string } <<
-                       { :name => :bytes_sent,      :type => :integer } <<
-                       { :name => :object_size,     :type => :integer } <<
+                       { :name => :bytes_sent,      :type => :traffic,  :unit => :byte } <<
+                       { :name => :object_size,     :type => :traffic,  :unit => :byte } <<
                        { :name => :total_time,      :type => :duration, :unit => :msec } <<
                        { :name => :turnaround_time, :type => :duration, :unit => :msec } <<
                        { :name => :referer,         :type => :referer } <<
@@ -34,7 +34,8 @@ module RequestLogAnalyzer::FileFormat
       analyze.hourly_spread
 
       analyze.frequency :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Most popular items"
-      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Duration"
+      analyze.duration :duration => :total_time, :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Request duration"
+      analyze.traffic  :traffic => :bytes_sent,  :category => lambda { |r| "#{r[:bucket]}/#{r[:key]}"}, :amount => 20, :title => "Traffic"
       analyze.frequency :category => :http_status, :title => 'HTTP status codes'
       analyze.frequency :category => :error_code, :title => 'Error codes'
     end
@@ -47,7 +48,7 @@ module RequestLogAnalyzer::FileFormat
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
-        d = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})/.match(value).captures
+        d = /^(\d{2})\/([A-Za-z]{3})\/(\d{4}).(\d{2}):(\d{2}):(\d{2})/.match(value).captures
         "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
       end
       

data/lib/request_log_analyzer/file_format/apache.rb

@@ -17,26 +17,30 @@ module RequestLogAnalyzer::FileFormat
     # A hash of predefined Apache log formats
     LOG_FORMAT_DEFAULTS = {
       :common   => '%h %l %u %t "%r" %>s %b',
-      :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
+      :combined => '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"',
+      :rack     => '%h %l %u %t "%r" %>s %b %T',
+      :referer  => '%{Referer}i -> %U',
+      :agent    => '%{User-agent}i'
     }
-
+    
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
       '%' => { :regexp => '%', :captures => [] },
       'h' => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] },
       'a' => { :regexp => '(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', :captures => [{:name => :remote_ip, :type => :string}] },
-      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :integer}] },
+      'b' => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] },
       'c' => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] },
-      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :msec}] },
+      'D' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :musec}] },
       'l' => { :regexp => '([\w-]+)', :captures => [{:name => :remote_logname, :type => :nillable_string}] },
-      'T' => { :regexp => '(\d+|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
-      't' => { :regexp => '\[([^\]]{26})\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
+      'T' => { :regexp => '((?:\d+(?:\.\d+))|-)', :captures => [{:name => :duration, :type => :duration, :unit => :sec}] },
+      't' => { :regexp => '\[(\d{2}\/[A-Za-z]{3}\/\d{4}.\d{2}:\d{2}:\d{2})(?: .\d{4})?\]', :captures => [{:name => :timestamp, :type => :timestamp}] },
       's' => { :regexp => '(\d{3})', :captures => [{:name => :http_status, :type => :integer}] },
       'u' => { :regexp => '(\w+|-)', :captures => [{:name => :user, :type => :nillable_string}] },
-      'r' => { :regexp => '([A-Z]+) ([^\s]+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
+      'U' => { :regexp => '(\/\S*)', :captures => [{:name => :path, :type => :string}] },
+      'r' => { :regexp => '([A-Z]+) (\S+) HTTP\/(\d+(?:\.\d+)*)', :captures => [{:name => :http_method, :type => :string},
                        {:name => :path, :type => :path}, {:name => :http_version, :type => :string}]},
-      'i' => { 'Referer'    => { :regexp => '([^\s]+)', :captures => [{:name => :referer, :type => :nillable_string}] },
-               'User-agent' => { :regexp => '(.*)',     :captures => [{:name => :user_agent, :type => :user_agent}] }
+      'i' => { 'Referer'    => { :regexp => '(\S+)', :captures => [{:name => :referer, :type => :nillable_string}] },
+               'User-agent' => { :regexp => '(.*)',  :captures => [{:name => :user_agent, :type => :user_agent}] }
              }
     }
 
@@ -97,6 +101,10 @@ module RequestLogAnalyzer::FileFormat
         analyze.duration :duration => :duration, :category => :path , :title => 'Request duration'
       end
 
+      if line_definition.captures?(:path) && line_definition.captures?(:bytes_sent)
+        analyze.traffic :traffic => :bytes_sent, :category => :path , :title => 'Traffic'
+      end
+
       return analyze.trackers
     end
 
@@ -109,7 +117,7 @@ module RequestLogAnalyzer::FileFormat
       # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
       # to speed up parsing.
       def convert_timestamp(value, definition)
-        d = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})/.match(value).captures
+        d = /^(\d{2})\/([A-Za-z]{3})\/(\d{4}).(\d{2}):(\d{2}):(\d{2})/.match(value).captures
         "#{d[2]}#{MONTHS[d[1]]}#{d[0]}#{d[3]}#{d[4]}#{d[5]}".to_i
       end
       

data/lib/request_log_analyzer/file_format/rack.rb

@@ -0,0 +1,11 @@
+module RequestLogAnalyzer::FileFormat
+  
+  class Rack < Apache
+    
+    def self.create(*args)
+      super(:rack, *args)
+    end
+    
+  end
+  
+end

data/lib/request_log_analyzer/output/fixed_width.rb

@@ -104,7 +104,7 @@ module RequestLogAnalyzer::Output
     end
 
     # Write a link
-    # <tt>text</tt> The text in the link
+    # <tt>text</tt> The text in the link, or the URL itself if no text is given
     # <tt>url</tt> The url to link to.
     def link(text, url = nil)
       if url.nil?
@@ -114,13 +114,13 @@ module RequestLogAnalyzer::Output
       end
     end
     
-    # Generate a header for a report    
+    # Generate a header for a report
     def header
       if io.kind_of?(File)
-        puts "Request-log-analyzer summary report"
-        line
+        puts colorize("Request-log-analyzer summary report", :white, :bold)
+        line(:green)
         puts "Version #{RequestLogAnalyzer::VERSION} - written by Willem van Bergen and Bart ten Brinke"
-        puts "Request-log-analyzer website: http://github.com/wvanbergen/request-log-analyzer"
+        puts "Website: #{link('http://github.com/wvanbergen/request-log-analyzer')}"
       end
     end
     
@@ -128,8 +128,9 @@ module RequestLogAnalyzer::Output
     def footer
       puts
       puts "Need an expert to analyze your application?"
-      puts "Mail to contact@railsdoctors.com or visit us at http://railsdoctors.com"
-      puts "Thanks for using request-log-analyzer!"
+      puts "Mail to #{link('contact@railsdoctors.com')} or visit us at #{link('http://railsdoctors.com')}."
+      line(:green)
+      puts "Thanks for using #{colorize('request-log-analyzer', :white, :bold)}!"
     end
     
     # Generate a report table and push it into the output object.

data/lib/request_log_analyzer/request.rb

@@ -41,13 +41,25 @@ module RequestLogAnalyzer
         DateTime.parse(value).strftime('%Y%m%d%H%M%S').to_i unless value.nil?
       end
       
+      def convert_traffic(value, capture_definition)
+        return nil if value.nil?
+        case capture_definition[:unit]
+        when :GB, :G, :gigabyte      then (value.to_f * 1000_000_000).round
+        when :GiB, :gibibyte         then (value.to_f * (2 ** 30)).round
+        when :MB, :M, :megabyte      then (value.to_f * 1000_000).round
+        when :MiB, :mebibyte         then (value.to_f * (2 ** 20)).round
+        when :KB, :K, :kilobyte, :kB then (value.to_f * 1000).round
+        when :KiB, :kibibyte         then (value.to_f * (2 ** 10)).round
+        else                              value.to_i
+        end
+      end
+      
       def convert_duration(value, capture_definition)
-        if value.nil?
-          nil
-        elsif capture_definition[:unit] == :msec
-          value.to_f / 1000.0      
-        else
-          value.to_f
+        return nil if value.nil?
+        case capture_definition[:unit]
+        when :microsec, :musec then value.to_f / 1000000.0
+        when :msec, :millisec  then value.to_f / 1000.0
+        else                        value.to_f
         end
       end
     end

data/lib/request_log_analyzer/tracker/duration.rb

@@ -51,10 +51,10 @@ module RequestLogAnalyzer::Tracker
           raise "Capture mismatch for multiple values in a request"
         end
       else
-        category = @categorizer.call(request)  # options[:category].respond_to?(:call) ? options[:category].call(request) : request[options[:category]]
-        duration = @durationizer.call(request) # options[:duration].respond_to?(:call) ? options[:duration].call(request) : request[options[:duration]]
+        category = @categorizer.call(request)
+        duration = @durationizer.call(request)
   
-        if duration.kind_of?(Float) && category.kind_of?(String)
+        if duration.kind_of?(Numeric) && !category.nil?
           @categories[category] ||= {:hits => 0, :cumulative => 0.0, :min => duration, :max => duration }
           @categories[category][:hits] += 1
           @categories[category][:cumulative] += duration

data/lib/request_log_analyzer/tracker/traffic.rb

@@ -0,0 +1,186 @@
+module RequestLogAnalyzer::Tracker
+
+  # Analyze the average and total traffic of requests
+  #
+  # === Options
+  # * <tt>:amount</tt> The amount of lines in the report
+  # * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
+  # * <tt>:traffic</tt> The field containing the duration in the request hash.
+  # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
+  # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
+  # * <tt>:title</tt> Title do be displayed above the report  
+  # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
+  class Traffic < Base
+    
+    attr_reader :categories
+
+    # Check if duration and catagory option have been received,
+    def prepare
+      raise "No traffic field set up for category tracker #{self.inspect}" unless options[:traffic]
+      raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
+
+      @categorizer = options[:category].respond_to?(:call) ? options[:category] : lambda { |request| request[options[:category]] }
+      @trafficizer = options[:traffic].respond_to?(:call)  ? options[:traffic]  : lambda { |request| request[options[:traffic]]  }
+      @categories = {}
+    end
+
+    # Get the duration information fron the request and store it in the different categories.
+    # <tt>request</tt> The request.
+    def update(request)
+      category = @categorizer.call(request)
+      traffic  = @trafficizer.call(request)
+
+      if traffic.kind_of?(Numeric) && !category.nil?
+        @categories[category] ||= {:hits => 0, :cumulative => 0, :min => traffic, :max => traffic }
+        @categories[category][:hits] += 1
+        @categories[category][:cumulative] += traffic
+        @categories[category][:min] = traffic if traffic < @categories[category][:min]
+        @categories[category][:max] = traffic if traffic > @categories[category][:max]
+      end
+    end
+    
+    # Get the number of hits of a specific category.
+    # <tt>cat</tt> The category
+    def hits(cat)
+      categories[cat][:hits]
+    end
+    
+    # Get the total duration of a specific category.
+    # <tt>cat</tt> The category
+    def cumulative_traffic(cat)
+      categories[cat][:cumulative]
+    end
+
+    # Get the minimal duration of a specific category.
+    # <tt>cat</tt> The category
+    def min_traffic(cat)
+      categories[cat][:min]
+    end
+
+    # Get the maximum duration of a specific category.
+    # <tt>cat</tt> The category
+    def max_traffic(cat)
+      categories[cat][:max]
+    end
+
+    # Get the average duration of a specific category.
+    # <tt>cat</tt> The category
+    def average_traffic(cat)
+      categories[cat][:cumulative].to_f / categories[cat][:hits]  
+    end
+    
+    # Get the average duration of a all categories.
+    def overall_average_traffic
+      overall_cumulative_duration.to_f / overall_hits
+    end
+    
+    # Get the cumlative duration of a all categories.
+    def overall_cumulative_traffic
+      categories.inject(0) { |sum, (name, cat)| sum + cat[:cumulative] }  
+    end
+    
+    # Get the total hits of a all categories.
+    def overall_hits
+      categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
+    end
+
+    # Return categories sorted by hits.
+    def sorted_by_hits
+      sorted_by(:hits)
+    end
+
+    # Return categories sorted by cumulative duration.
+    def sorted_by_cumulative
+      sorted_by(:cumulative)
+    end
+
+    # Return categories sorted by cumulative duration.
+    def sorted_by_average
+      sorted_by { |cat| cat[:cumulative].to_f / cat[:hits] }
+    end
+          
+    # Return categories sorted by a given key.
+    # <tt>by</tt> The key.
+    def sorted_by(by = nil) 
+      if block_given?
+        categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) } 
+      else
+        categories.sort { |a, b| b[1][by] <=> a[1][by] } 
+      end
+    end
+    
+    # Block function to build a result table using a provided sorting function.
+    # <tt>output</tt> The output object.
+    # <tt>amount</tt> The number of rows in the report table (default 10).
+    # === Options
+    #  * </tt>:title</tt> The title of the table
+    #  * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
+    def report_table(output, amount = 10, options = {}, &block)
+      
+      output.title(options[:title])
+      
+      top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
+      output.table({:title => 'Category', :width => :rest}, 
+            {:title => 'Hits',       :align => :right, :highlight => (options[:sort] == :hits), :min_width => 4}, 
+            {:title => 'Cumulative', :align => :right, :highlight => (options[:sort] == :cumulative), :min_width => 10}, 
+            {:title => 'Average',    :align => :right, :highlight => (options[:sort] == :average), :min_width => 8},
+            {:title => 'Min',        :align => :right, :highlight => (options[:sort] == :min)}, 
+            {:title => 'Max',        :align => :right, :highlight => (options[:sort] == :max)}) do |rows|
+      
+        top_categories.each do |(cat, info)|
+          rows << [cat, info[:hits], format_traffic(info[:cumulative]), format_traffic((info[:cumulative] / info[:hits]).round),
+                    format_traffic(info[:min]), format_traffic(info[:max])]
+        end
+      end
+    end
+    
+    # Formats the traffic number using x B/kB/MB/GB etc notation
+    def format_traffic(bytes)
+      return "0 B" if bytes.zero?
+      case Math.log10(bytes).floor
+      when  1...4  then '%d B'  % bytes
+      when  4...7  then '%d kB' % (bytes / 1000)
+      when  7...10 then '%d MB' % (bytes / 1000_000)
+      when 10...13 then '%d GB' % (bytes / 1000_000_000)
+      else              '%d TB' % (bytes / 1000_000_000_000)
+      end
+    end
+
+    # Generate a request duration report to the given output object
+    # By default colulative and average duration are generated.
+    # Any options for the report should have been set during initialize.
+    # <tt>output</tt> The output object
+    def report(output)
+
+      options[:report] ||= [:cumulative, :average]
+      options[:top]    ||= 20
+  
+      options[:report].each do |report|
+        case report
+        when :average
+          report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by average", :sort => :average) { |cat| cat[:cumulative] / cat[:hits] }  
+        when :cumulative
+          report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by sum", :sort => :cumulative) { |cat| cat[:cumulative] }
+        when :hits
+          report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by hits", :sort => :hits) { |cat| cat[:hits] }
+        else
+          raise "Unknown duration report specified: #{report}!"
+        end
+      end
+      
+      output.puts
+      output.puts "#{output.colorize(title, :white, :bold)} - observed total: " + output.colorize(format_traffic(overall_cumulative_traffic), :brown, :bold)
+    end
+    
+    # Returns the title of this tracker for reports
+    def title
+      options[:title]  || 'Request traffic'
+    end
+    
+    # Returns all the categories and the tracked duration as a hash than can be exported to YAML
+    def to_yaml_object
+      return nil if @categories.empty?
+      @categories
+    end
+  end
+end

data/lib/request_log_analyzer.rb

@@ -10,8 +10,8 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
 module RequestLogAnalyzer
   
   # The current version of request-log-analyzer.
-  # This will be diplayed in output reports etc.  
-  VERSION = "1.3.4"
+  # This will be diplayed in output reports etc.
+  VERSION = "1.3.5"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
@@ -25,7 +25,7 @@ module RequestLogAnalyzer
   # <tt>const</tt>:: The constant to load from the base constant as a string or symbol. This should be 'Bar' or :Bar when the constant Foo::Bar is being loaded.
   def self.load_default_class_file(base, const)
     require "#{to_underscore("#{base.name}::#{const}")}"
-    base.const_get(const)
+    base.const_get(const) if base.const_defined?(const)
   end
 
   # Convert a string/symbol in camelcase (RequestLogAnalyzer::Controller) to underscores (request_log_analyzer/controller)

data/request-log-analyzer.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
-  s.version = "1.3.4"
-  s.date    = "2009-09-14"
+  s.version = "1.3.5"
+  s.date    = "2009-09-16"
   
   s.rubyforge_project = 'r-l-a'
   
@@ -31,6 +31,6 @@ Gem::Specification.new do |s|
   s.email    = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
   s.homepage = 'http://railsdoctors.com'
 
-  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log spec/lib/macros.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log lib/request_log_analyzer/file_format/amazon_s3.rb tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/database.rb spec/fixtures/decompression.log.bz2 lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb spec/fixtures/rails_unordered.log bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/testing_format.rb spec/fixtures/test_order.log spec/fixtures/rails.db lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb lib/request_log_analyzer/mailer.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb spec/database.yml Rakefile lib/request_log_analyzer/database/connection.rb spec/unit/filter/filter_spec.rb spec/fixtures/test_language_combined.log lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/decompression.tar.gz spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb lib/cli/database_console.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/lib/mocks.rb spec/lib/helpers.rb lib/cli/database_console_init.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/fixtures/rails_1x.log spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log tasks/github-gem.rake spec/unit/database/database_spec.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log spec/fixtures/syslog_1x.log LICENSE lib/request_log_analyzer/source/database_loader.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
-  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
+  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log spec/lib/macros.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log lib/request_log_analyzer/file_format/amazon_s3.rb tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/database.rb spec/fixtures/decompression.log.bz2 spec/fixtures/rails_unordered.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/testing_format.rb spec/fixtures/test_order.log spec/fixtures/rails.db lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb lib/request_log_analyzer/mailer.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb spec/database.yml Rakefile lib/request_log_analyzer/database/connection.rb spec/unit/filter/filter_spec.rb spec/fixtures/test_language_combined.log lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb lib/cli/database_console.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/lib/mocks.rb spec/lib/helpers.rb lib/cli/database_console_init.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/fixtures/rails_1x.log spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log tasks/github-gem.rake spec/unit/database/database_spec.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/traffic.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/syslog_1x.log spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log LICENSE lib/request_log_analyzer/source/database_loader.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
+  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
 end

data/spec/lib/mocks.rb

@@ -25,7 +25,7 @@ module RequestLogAnalyzer::Spec::Mocks
   def mock_io
     mio = mock('IO')
     mio.stub!(:print)
-    mio.stub!(:puts)    
+    mio.stub!(:puts)
     mio.stub!(:write)
     return mio
   end
@@ -35,12 +35,15 @@ module RequestLogAnalyzer::Spec::Mocks
     output.stub!(:header)
     output.stub!(:footer)   
     output.stub!(:puts)
-    output.stub!(:<<)    
+    output.stub!(:<<)
+    output.stub!(:colorize).and_return("Fancy text")
+    output.stub!(:link)
     output.stub!(:title)
     output.stub!(:line)
     output.stub!(:with_style)    
     output.stub!(:table).and_yield([])
     output.stub!(:io).and_return(mock_io)
+    
     return output
   end
   

data/spec/unit/file_format/apache_format_spec.rb

@@ -99,12 +99,49 @@ describe RequestLogAnalyzer::FileFormat::Apache do
       @log_parser.parse_file(log_fixture(:apache_common)) { counter.hit! }
     end
   end
+  
+  context '"Rack" access log parser' do
+    before(:each) do
+      @file_format = RequestLogAnalyzer::FileFormat.load(:rack)
+      @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+      @sample_1 = '127.0.0.1 - - [16/Sep/2009 06:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453'
+    end
+    
+    it "should create a kind of an Apache file format" do
+      @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Apache)
+    end
+    
+    it "should have a valid language definitions" do
+      @file_format.should be_valid
+    end
+
+    it "should parse a valid access log line" do
+      @file_format.line_definitions[:access].matches(@sample_1).should be_kind_of(Hash)
+    end
+
+    it "should not parse a valid access log line" do
+      @file_format.line_definitions[:access].matches('addasdsasadadssadasd').should be_false
+    end
+
+    it "should read the correct values from a valid 404 access log line" do
+      @log_parser.parse_io(StringIO.new(@sample_1)) do |request|
+        request[:remote_host].should  == '127.0.0.1'
+        request[:timestamp].should    == 20090916064008
+        request[:http_status].should  == 500
+        request[:http_method].should  == 'GET'
+        request[:http_version].should == '1.1'
+        request[:bytes_sent].should   == 63183
+        request[:user].should         == nil
+        request[:duration].should     == 0.0453
+      end
+    end
+  end
  
   context '"Combined" access log parsing' do
 
     before(:all) do
       @file_format = RequestLogAnalyzer::FileFormat.load(:apache, :combined)
-      @log_parser = RequestLogAnalyzer::Source::LogParser.new(@file_format)
+      @log_parser  = RequestLogAnalyzer::Source::LogParser.new(@file_format)
       @sample_1 = '69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)"'
       @sample_2 = '10.0.1.1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9"'
     end

data/spec/unit/tracker/traffic_tracker_spec.rb

@@ -0,0 +1,105 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe RequestLogAnalyzer::Tracker::Traffic do
+  
+  describe '#update' do
+    
+    context 'using a field-based category' do
+      before(:each) do
+        @tracker = RequestLogAnalyzer::Tracker::Traffic.new(:traffic => :traffic, :category => :category)
+        @tracker.prepare
+      end
+    
+      it "should register a request in the right category" do
+        @tracker.update(request(:category => 'a', :traffic => 200))
+        @tracker.categories.should include('a')
+      end
+    
+      it "should register a hit in the right category" do
+        @tracker.update(request(:category => 'a', :traffic => 1))
+        @tracker.update(request(:category => 'b', :traffic => 2))
+        @tracker.update(request(:category => 'b', :traffic => 3))
+      
+        @tracker.hits('a').should == 1
+        @tracker.hits('b').should == 2
+      end
+    
+      it "should sum the traffics of the same category as cumulative traffic" do
+        @tracker.update(request(:category => 'a', :traffic => 1))
+        @tracker.update(request(:category => 'b', :traffic => 2))
+        @tracker.update(request(:category => 'b', :traffic => 3))
+      
+        @tracker.cumulative_traffic('a').should == 1
+        @tracker.cumulative_traffic('b').should == 5
+      end
+    
+      it "should calculate the average traffic correctly" do
+        @tracker.update(request(:category => 'a', :traffic => 1))
+        @tracker.update(request(:category => 'b', :traffic => 2))
+        @tracker.update(request(:category => 'b', :traffic => 3))
+      
+        @tracker.average_traffic('a').should == 1.0
+        @tracker.average_traffic('b').should == 2.5
+      end
+    
+      it "should set min and max traffic correctly" do
+        @tracker.update(request(:category => 'a', :traffic => 1))
+        @tracker.update(request(:category => 'b', :traffic => 2))
+        @tracker.update(request(:category => 'b', :traffic => 3))
+      
+        @tracker.min_traffic('b').should == 2
+        @tracker.max_traffic('b').should == 3
+      end  
+    
+    end
+  
+    context 'using a dynamic category' do
+      before(:each) do
+        @categorizer = Proc.new { |request| request[:traffic] < 2 ? 'few' : 'lots' }
+        @tracker = RequestLogAnalyzer::Tracker::Traffic.new(:traffic => :traffic, :category => @categorizer)
+        @tracker.prepare
+      end
+    
+      it "should use the categorizer to determine the right category" do
+        @tracker.update(request(:category => 'a', :traffic => 1))
+        @tracker.update(request(:category => 'b', :traffic => 2))
+        @tracker.update(request(:category => 'b', :traffic => 3))
+      
+        @tracker.categories.should include('few', 'lots')
+      end
+    
+      it "should use the categorizer to aggregate the values correctly" do
+        @tracker.update(request(:category => 'a', :traffic => 1))
+        @tracker.update(request(:category => 'b', :traffic => 2))
+        @tracker.update(request(:category => 'b', :traffic => 3))
+      
+        @tracker.max_traffic('few').should == 1
+        @tracker.min_traffic('lots').should == 2
+      end
+    end
+  end
+  
+  describe '#report' do
+    before(:each) do
+      @tracker = RequestLogAnalyzer::Tracker::Traffic.new(:category => :category, :traffic => :traffic)
+      @tracker.prepare
+    end
+    
+    it "should generate a report without errors when one category is present" do
+      @tracker.update(request(:category => 'a', :traffic => 2))
+      @tracker.report(mock_output)
+      lambda { @tracker.report(mock_output) }.should_not raise_error
+    end
+  
+    it "should generate a report without errors when no category is present" do
+      lambda { @tracker.report(mock_output) }.should_not raise_error
+    end
+  
+    it "should generate a report without errors when multiple categories are present" do
+      @tracker.update(request(:category => 'a', :traffic => 2))
+      @tracker.update(request(:category => 'b', :traffic => 2))
+      lambda { @tracker.report(mock_output) }.should_not raise_error
+    end
+    
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: request-log-analyzer
 version: !ruby/object:Gem::Version 
-  version: 1.3.4
+  version: 1.3.5
 platform: ruby
 authors: 
 - Willem van Bergen
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-09-14 00:00:00 +02:00
+date: 2009-09-16 00:00:00 +02:00
 default_executable: request-log-analyzer
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -61,10 +61,10 @@ files:
 - spec/integration/command_line_usage_spec.rb
 - lib/request_log_analyzer/database.rb
 - spec/fixtures/decompression.log.bz2
+- spec/fixtures/rails_unordered.log
 - lib/request_log_analyzer/log_processor.rb
 - lib/request_log_analyzer/tracker.rb
 - lib/request_log_analyzer/filter.rb
-- spec/fixtures/rails_unordered.log
 - bin/request-log-analyzer
 - request-log-analyzer.gemspec
 - DESIGN.rdoc
@@ -103,8 +103,10 @@ files:
 - spec/fixtures/test_language_combined.log
 - lib/request_log_analyzer/aggregator/database_inserter.rb
 - lib/request_log_analyzer/aggregator/summarizer.rb
+- lib/request_log_analyzer/file_format/rack.rb
 - lib/request_log_analyzer/file_format/rails.rb
 - spec/fixtures/decompression.tar.gz
+- spec/unit/tracker/traffic_tracker_spec.rb
 - spec/unit/filter/field_filter_spec.rb
 - spec/unit/database/base_class_spec.rb
 - lib/request_log_analyzer/filter/timespan.rb
@@ -136,11 +138,12 @@ files:
 - tasks/github-gem.rake
 - spec/unit/database/database_spec.rb
 - lib/request_log_analyzer/tracker/duration.rb
+- lib/request_log_analyzer/tracker/traffic.rb
 - lib/request_log_analyzer/file_format.rb
 - spec/unit/aggregator/summarizer_spec.rb
+- spec/fixtures/syslog_1x.log
 - spec/fixtures/rails_22.log
 - spec/fixtures/multiple_files_2.log
-- spec/fixtures/syslog_1x.log
 - LICENSE
 - lib/request_log_analyzer/source/database_loader.rb
 - spec/unit/tracker/frequency_tracker_spec.rb
@@ -191,6 +194,7 @@ test_files:
 - spec/unit/file_format/amazon_s3_format_spec.rb
 - spec/unit/controller/log_processor_spec.rb
 - spec/unit/filter/filter_spec.rb
+- spec/unit/tracker/traffic_tracker_spec.rb
 - spec/unit/filter/field_filter_spec.rb
 - spec/unit/database/base_class_spec.rb
 - spec/unit/tracker/timespan_tracker_spec.rb