request-log-analyzer 1.3.5 → 1.4.0

data/lib/request_log_analyzer/tracker/traffic.rb

@@ -3,139 +3,58 @@ module RequestLogAnalyzer::Tracker
   # Analyze the average and total traffic of requests
   #
   # === Options
-  # * <tt>:amount</tt> The amount of lines in the report
   # * <tt>:category</tt> Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER)
   # * <tt>:traffic</tt> The field containing the duration in the request hash.
   # * <tt>:if</tt> Proc that has to return !nil for a request to be passed to the tracker.
   # * <tt>:line_type</tt> The line type that contains the duration field (determined by the category proc).
-  # * <tt>:title</tt> Title do be displayed above the report  
+  # * <tt>:title</tt> Title do be displayed above the report
   # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
   class Traffic < Base
-    
+
     attr_reader :categories
 
+    include RequestLogAnalyzer::Tracker::StatisticsTracking
+
     # Check if duration and catagory option have been received,
     def prepare
       raise "No traffic field set up for category tracker #{self.inspect}" unless options[:traffic]
-      raise "No categorizer set up for duration tracker #{self.inspect}" unless options[:category]
+      raise "No categorizer set up for duration tracker #{self.inspect}"   unless options[:category]
 
-      @categorizer = options[:category].respond_to?(:call) ? options[:category] : lambda { |request| request[options[:category]] }
-      @trafficizer = options[:traffic].respond_to?(:call)  ? options[:traffic]  : lambda { |request| request[options[:traffic]]  }
-      @categories = {}
+      @categorizer = create_lambda(options[:category])
+      @trafficizer = create_lambda(options[:traffic])
+      @categories  = {}
     end
 
-    # Get the duration information fron the request and store it in the different categories.
+
+    # Get the duration information from the request and store it in the different categories.
     # <tt>request</tt> The request.
     def update(request)
       category = @categorizer.call(request)
       traffic  = @trafficizer.call(request)
-
-      if traffic.kind_of?(Numeric) && !category.nil?
-        @categories[category] ||= {:hits => 0, :cumulative => 0, :min => traffic, :max => traffic }
-        @categories[category][:hits] += 1
-        @categories[category][:cumulative] += traffic
-        @categories[category][:min] = traffic if traffic < @categories[category][:min]
-        @categories[category][:max] = traffic if traffic > @categories[category][:max]
-      end
-    end
-    
-    # Get the number of hits of a specific category.
-    # <tt>cat</tt> The category
-    def hits(cat)
-      categories[cat][:hits]
-    end
-    
-    # Get the total duration of a specific category.
-    # <tt>cat</tt> The category
-    def cumulative_traffic(cat)
-      categories[cat][:cumulative]
-    end
-
-    # Get the minimal duration of a specific category.
-    # <tt>cat</tt> The category
-    def min_traffic(cat)
-      categories[cat][:min]
-    end
-
-    # Get the maximum duration of a specific category.
-    # <tt>cat</tt> The category
-    def max_traffic(cat)
-      categories[cat][:max]
-    end
-
-    # Get the average duration of a specific category.
-    # <tt>cat</tt> The category
-    def average_traffic(cat)
-      categories[cat][:cumulative].to_f / categories[cat][:hits]  
-    end
-    
-    # Get the average duration of a all categories.
-    def overall_average_traffic
-      overall_cumulative_duration.to_f / overall_hits
-    end
-    
-    # Get the cumlative duration of a all categories.
-    def overall_cumulative_traffic
-      categories.inject(0) { |sum, (name, cat)| sum + cat[:cumulative] }  
-    end
-    
-    # Get the total hits of a all categories.
-    def overall_hits
-      categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
+      update_statistics(category, traffic) if traffic.kind_of?(Numeric) && !category.nil?
     end
 
-    # Return categories sorted by hits.
-    def sorted_by_hits
-      sorted_by(:hits)
-    end
-
-    # Return categories sorted by cumulative duration.
-    def sorted_by_cumulative
-      sorted_by(:cumulative)
-    end
-
-    # Return categories sorted by cumulative duration.
-    def sorted_by_average
-      sorted_by { |cat| cat[:cumulative].to_f / cat[:hits] }
-    end
-          
-    # Return categories sorted by a given key.
-    # <tt>by</tt> The key.
-    def sorted_by(by = nil) 
-      if block_given?
-        categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) } 
-      else
-        categories.sort { |a, b| b[1][by] <=> a[1][by] } 
-      end
-    end
-    
     # Block function to build a result table using a provided sorting function.
     # <tt>output</tt> The output object.
     # <tt>amount</tt> The number of rows in the report table (default 10).
     # === Options
     #  * </tt>:title</tt> The title of the table
     #  * </tt>:sort</tt> The key to sort on (:hits, :cumulative, :average, :min or :max)
-    def report_table(output, amount = 10, options = {}, &block)
-      
-      output.title(options[:title])
-      
-      top_categories = @categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }.slice(0...amount)
-      output.table({:title => 'Category', :width => :rest}, 
-            {:title => 'Hits',       :align => :right, :highlight => (options[:sort] == :hits), :min_width => 4}, 
-            {:title => 'Cumulative', :align => :right, :highlight => (options[:sort] == :cumulative), :min_width => 10}, 
-            {:title => 'Average',    :align => :right, :highlight => (options[:sort] == :average), :min_width => 8},
-            {:title => 'Min',        :align => :right, :highlight => (options[:sort] == :min)}, 
-            {:title => 'Max',        :align => :right, :highlight => (options[:sort] == :max)}) do |rows|
-      
-        top_categories.each do |(cat, info)|
-          rows << [cat, info[:hits], format_traffic(info[:cumulative]), format_traffic((info[:cumulative] / info[:hits]).round),
-                    format_traffic(info[:min]), format_traffic(info[:max])]
+    def report_table(output, sort, options = {}, &block)
+      output.puts
+
+      top_categories = output.slice_results(sorted_by(sort))
+      output.with_style(:top_line => true) do      
+        output.table(*statistics_header(:title => options[:title],:highlight => sort)) do |rows|
+          top_categories.each { |(cat, info)| rows.push(statistics_row(cat)) }
         end
       end
+      output.puts
     end
-    
+
     # Formats the traffic number using x B/kB/MB/GB etc notation
-    def format_traffic(bytes)
+    def display_value(bytes)
+      return "-"   if bytes.nil?
       return "0 B" if bytes.zero?
       case Math.log10(bytes).floor
       when  1...4  then '%d B'  % bytes
@@ -152,31 +71,32 @@ module RequestLogAnalyzer::Tracker
     # <tt>output</tt> The output object
     def report(output)
 
-      options[:report] ||= [:cumulative, :average]
-      options[:top]    ||= 20
-  
-      options[:report].each do |report|
+      sortings = output.options[:sort] || [:sum, :mean]
+
+      sortings.each do |report|
         case report
-        when :average
-          report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by average", :sort => :average) { |cat| cat[:cumulative] / cat[:hits] }  
-        when :cumulative
-          report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by sum", :sort => :cumulative) { |cat| cat[:cumulative] }
+        when :mean
+          report_table(output, :mean,   :title => "#{title} - sorted by mean")
+        when :stddev
+          report_table(output, :stddev, :title => "#{title} - sorted by standard deviation")
+        when :sum
+          report_table(output, :sum,    :title => "#{title} - sorted by sum")
         when :hits
-          report_table(output, options[:top], :title => "#{title} - top #{options[:top]} by hits", :sort => :hits) { |cat| cat[:hits] }
+          report_table(output, :hits,   :title => "#{title} - sorted by hits")
         else
           raise "Unknown duration report specified: #{report}!"
         end
       end
-      
+
       output.puts
-      output.puts "#{output.colorize(title, :white, :bold)} - observed total: " + output.colorize(format_traffic(overall_cumulative_traffic), :brown, :bold)
+      output.puts "#{output.colorize(title, :white, :bold)} - observed total: " + output.colorize(display_value(sum_overall), :brown, :bold)
     end
-    
+
     # Returns the title of this tracker for reports
     def title
       options[:title]  || 'Request traffic'
     end
-    
+
     # Returns all the categories and the tracked duration as a hash than can be exported to YAML
     def to_yaml_object
       return nil if @categories.empty?

data/lib/request_log_analyzer/tracker.rb

@@ -16,84 +16,191 @@ module RequestLogAnalyzer::Tracker
   #
   # For example :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
   class Base
-    
+
     attr_reader :options
-    
+
     # Initialize the class
     # Note that the options are only applicable if should_update? is not overwritten
     # by the inheriting class.
-    # 
+    #
     # === Options
     # * <tt>:if</tt> Handle request if this proc is true for the handled request.
     # * <tt>:unless</tt> Handle request if this proc is false for the handled request.
     # * <tt>:line_type</tt> Line type this tracker will accept.
     def initialize(options ={})
       @options = options
+      setup_should_update_checks!
+    end
+
+    # Sets up the tracker's should_update? checks.
+    def setup_should_update_checks!
+      @should_update_checks = []
+      @should_update_checks.push( lambda { |request| request.has_line_type?(options[:line_type]) } ) if options[:line_type]
+      @should_update_checks.push(options[:if]) if options[:if].respond_to?(:call)
+      @should_update_checks.push( lambda { |request| request[options[:if]] }) if options[:if].kind_of?(Symbol)
+      @should_update_checks.push( lambda { |request| !options[:unless].call(request) }) if options[:unless].respond_to?(:call)
+      @should_update_checks.push( lambda { |request| !request[options[:unless]] }) if options[:unless].kind_of?(Symbol)
     end
     
+    # Creates a lambda expression to return a static field from a request. If the 
+    # argument already is a lambda exprssion, it will simply return the argument.
+    def create_lambda(arg)
+      case arg
+      when Proc   then arg
+      when Symbol then lambda { |request| request[arg] }
+      else raise "Canot create a lambda expression from this argument: #{arg.inspect}!"
+      end
+    end
+
     # Hook things that need to be done before running here.
     def prepare
     end
-    
+
     # Will be called with each request.
     # <tt>request</tt> The request to track data in.
     def update(request)
     end
-    
+
     # Hook things that need to be done after running here.
     def finalize
     end
-    
+
     # Determine if we should run the update function at all.
     # Usually the update function will be heavy, so a light check is done here
     # determining if we need to call update at all.
     #
-    # Default this checks if defined: 
+    # Default this checks if defined:
     #  * :line_type is also in the request hash.
     #  * :if is true for this request.
     #  * :unless if false for this request
     #
     # <tt>request</tt> The request object.
     def should_update?(request)
-      return false if options[:line_type] && !request.has_line_type?(options[:line_type])
-      
-      if options[:if]
-        if options[:if].kind_of?(Symbol)
-          return false unless request[options[:if]]
-        elsif options[:if].respond_to?(:call)
-          return false unless options[:if].call(request)
-        end
-      end
-      
-      if options[:unless]
-        if options[:unless].kind_of?(Symbol)
-          return false if request[options[:unless]]
-        elsif options[:unless].respond_to?(:call)
-          return false if options[:unless].call(request)
-        end
-      end
-      
-      return true
+      @should_update_checks.all? { |c| c.call(request) }
     end
-    
+
     # Hook report generation here.
     # Defaults to self.inspect
     # <tt>output</tt> The output object the report will be passed to.
     def report(output)
       output << self.inspect
-      output << "\n"  
+      output << "\n"
     end
-    
+
     # The title of this tracker. Used for reporting.
     def title
       self.class.to_s
     end
-    
+
     # This method is called by RequestLogAnalyzer::Aggregator:Summarizer to retrieve an
     # object with all the results of this tracker, that can be dumped to YAML format.
     def to_yaml_object
       nil
     end
-         
-  end 
+  end
+  
+  module StatisticsTracking
+    
+    # Update sthe running calculation of statistics with the newly found numeric value.
+    # <tt>category</tt>:: The category for which to update the running statistics calculations
+    # <tt>number</tt>:: The numeric value to update the calculations with.
+    def update_statistics(category, number)
+      @categories[category] ||= {:hits => 0, :sum => 0, :mean => 0.0, :sum_of_squares => 0.0, :min => number, :max => number }
+      delta = number - @categories[category][:mean]
+
+      @categories[category][:hits]           += 1
+      @categories[category][:mean]           += (delta / @categories[category][:hits])
+      @categories[category][:sum_of_squares] += delta * (number - @categories[category][:mean])
+      @categories[category][:sum]            += number
+      @categories[category][:min]             = number if number < @categories[category][:min]
+      @categories[category][:max]             = number if number > @categories[category][:max]
+    end
+    
+    # Get the number of hits of a specific category.
+    # <tt>cat</tt> The category
+    def hits(cat)
+      @categories[cat][:hits]
+    end
+
+    # Get the total duration of a specific category.
+    # <tt>cat</tt> The category
+    def sum(cat)
+      @categories[cat][:sum]
+    end
+
+    # Get the minimal duration of a specific category.
+    # <tt>cat</tt> The category
+    def min(cat)
+      @categories[cat][:min]
+    end
+
+    # Get the maximum duration of a specific category.
+    # <tt>cat</tt> The category
+    def max(cat)
+      @categories[cat][:max]
+    end
+
+    # Get the average duration of a specific category.
+    # <tt>cat</tt> The category
+    def mean(cat)
+      @categories[cat][:mean]
+    end
+
+    # Get the standard deviation of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def stddev(cat)
+      Math.sqrt(variance(cat))
+    end
+
+    # Get the variance of the duration of a specific category.
+    # <tt>cat</tt> The category
+    def variance(cat)
+      return 0.0 if @categories[cat][:hits] <= 1
+      (@categories[cat][:sum_of_squares] / (@categories[cat][:hits] - 1))
+    end
+
+    # Get the average duration of a all categories.
+    def mean_overall
+      sum_overall / hits_overall
+    end
+
+    # Get the cumlative duration of a all categories.
+    def sum_overall
+      @categories.inject(0.0) { |sum, (name, cat)| sum + cat[:sum] }
+    end
+
+    # Get the total hits of a all categories.
+    def hits_overall
+      @categories.inject(0) { |sum, (name, cat)| sum + cat[:hits] }
+    end
+    
+    # Return categories sorted by a given key.
+    # <tt>by</tt> The key to sort on. This parameter can be omitted if a sorting block is provided instead
+    def sorted_by(by = nil)
+      if block_given?
+        categories.sort { |a, b| yield(b[1]) <=> yield(a[1]) }
+      else
+        categories.sort { |a, b| send(by, b[0]) <=> send(by, a[0]) }
+      end
+    end
+    
+    # Returns the column header for a statistics table to report on the statistics result
+    def statistics_header(options)
+      [
+        {:title => options[:title], :width => :rest},
+        {:title => 'Hits',   :align => :right, :highlight => (options[:highlight] == :hits),   :min_width => 4},
+        {:title => 'Sum',    :align => :right, :highlight => (options[:highlight] == :sum),    :min_width => 6},
+        {:title => 'Mean',   :align => :right, :highlight => (options[:highlight] == :mean),   :min_width => 6},
+        {:title => 'StdDev', :align => :right, :highlight => (options[:highlight] == :stddev), :min_width => 6},
+        {:title => 'Min',    :align => :right, :highlight => (options[:highlight] == :min),    :min_width => 6},
+        {:title => 'Max',    :align => :right, :highlight => (options[:highlight] == :max),    :min_width => 6}
+      ]
+    end
+    
+    # Returns a row of statistics information for a report table, given a category
+    def statistics_row(cat)
+      [cat, hits(cat), display_value(sum(cat)), display_value(mean(cat)), display_value(stddev(cat)),
+                display_value(min(cat)), display_value(max(cat))]
+    end
+  end
 end

data/lib/request_log_analyzer.rb

@@ -8,10 +8,10 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
 # - This module itselfs contains some functions to help with class and source file loading.
 # - The actual application resides in the RequestLogAnalyzer::Controller class.
 module RequestLogAnalyzer
-  
+
   # The current version of request-log-analyzer.
-  # This will be diplayed in output reports etc.
-  VERSION = "1.3.5"
+  # Do not change the value by hand; it will be updated automatically by the gem release script.
+  VERSION = "1.4.0"
 
   # Loads constants in the RequestLogAnalyzer namespace using self.load_default_class_file(base, const)
   # <tt>const</tt>:: The constant that is not yet loaded in the RequestLogAnalyzer namespace. This should be passed as a string or symbol.
@@ -35,7 +35,7 @@ module RequestLogAnalyzer
     str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
   end
 
-  # Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase 
+  # Convert a string/symbol in underscores (<tt>request_log_analyzer/controller</tt>) to camelcase
   # (<tt>RequestLogAnalyzer::Controller</tt>). This can be used to find the class that is defined in a given filename.
   # <tt>str</tt>:: The string to convert in the following format: <tt>module_name/class_name</tt>
   def self.to_camelcase(str)

data/request-log-analyzer.gemspec

@@ -1,36 +1,40 @@
 Gem::Specification.new do |s|
   s.name    = "request-log-analyzer"
-  s.version = "1.3.5"
-  s.date    = "2009-09-16"
   
+  # Do not set the version and date field manually, this is done by the release script
+  s.version = "1.4.0"
+  s.date    = "2009-09-30"
+
   s.rubyforge_project = 'r-l-a'
-  
+
   s.bindir             = 'bin'
   s.executables        = ['request-log-analyzer']
   s.default_executable = 'request-log-analyzer'
-  
-  s.summary     = "A command line tool to analyze request logs for Apache, Rails, Merb and other application servers"
+
+  s.summary     = "A command line tool to analyze request logs for Apache, Rails, Merb and other web application servers"
   s.description = <<-eos
     Request log analyzer's purpose is to find ot how your web application is being used and to focus your optimization efforts.
-    This tool will parse all requests in the application's log file and aggregate the information. Once it is finished parsing 
-    the log file(s), it will show the requests that take op most server time using various metrics. It can also insert all 
-    parsed request information into a database so you can roll your own analysis. It supports Rails- and Merb-based applications 
-    and Apache access log files out of the box, but file formats of other applications can easily be supported by supplying an 
+    This tool will parse all requests in the application's log file and aggregate the information. Once it is finished parsing
+    the log file(s), it will show the requests that take op most server time using various metrics. It can also insert all
+    parsed request information into a database so you can roll your own analysis. It supports Rails- and Merb-based applications
+    and Apache access log files out of the box, but file formats of other applications can easily be supported by supplying an
     easy to write log file format definition.
   eos
-  
+
   s.rdoc_options << '--title' << s.name << '--main' << 'README.rdoc' << '--line-numbers' << '--inline-source'
   s.extra_rdoc_files = ['README.rdoc']
-  
+
   s.requirements << "To use the database inserter, ActiveRecord and an appropriate database adapter are required."
-  
+
   s.add_development_dependency('rspec', '>= 1.2.4')
   s.add_development_dependency('git',   '>= 1.1.0')
-  
+
   s.authors  = ['Willem van Bergen', 'Bart ten Brinke']
   s.email    = ['willem@railsdoctors.com', 'bart@railsdoctors.com']
   s.homepage = 'http://railsdoctors.com'
 
-  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/fixtures/rails_22_cached.log spec/lib/macros.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log lib/request_log_analyzer/file_format/amazon_s3.rb tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/database.rb spec/fixtures/decompression.log.bz2 spec/fixtures/rails_unordered.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/testing_format.rb spec/fixtures/test_order.log spec/fixtures/rails.db lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/aggregator.rb lib/cli/progressbar.rb lib/request_log_analyzer/mailer.rb README.rdoc spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb spec/database.yml Rakefile lib/request_log_analyzer/database/connection.rb spec/unit/filter/filter_spec.rb spec/fixtures/test_language_combined.log lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb lib/cli/database_console.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/lib/mocks.rb spec/lib/helpers.rb lib/cli/database_console_init.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/fixtures/rails_1x.log spec/fixtures/decompression.log.zip spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log tasks/github-gem.rake spec/unit/database/database_spec.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/traffic.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/syslog_1x.log spec/fixtures/rails_22.log spec/fixtures/multiple_files_2.log LICENSE lib/request_log_analyzer/source/database_loader.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
-  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/request_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
+  # The files and test_files directives are set automatically by the release script.
+  # Do not change them by hand, but make sure to add the files to the git repository.
+  s.files      = %w(spec/unit/filter/anonymize_filter_spec.rb spec/fixtures/rails_22_cached.log lib/request_log_analyzer/line_definition.rb lib/request_log_analyzer/output/html.rb lib/request_log_analyzer/controller.rb spec/lib/macros.rb lib/request_log_analyzer/file_format/rails_development.rb spec/fixtures/apache_combined.log spec/fixtures/apache_common.log spec/fixtures/merb_prefixed.log lib/request_log_analyzer/file_format/amazon_s3.rb tasks/request_log_analyzer.rake spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb lib/request_log_analyzer/database.rb spec/fixtures/decompression.log.bz2 spec/fixtures/rails_unordered.log lib/request_log_analyzer/log_processor.rb lib/request_log_analyzer/tracker.rb lib/request_log_analyzer/filter.rb bin/request-log-analyzer request-log-analyzer.gemspec DESIGN.rdoc spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/lib/matchers.rb lib/request_log_analyzer/filter/field.rb lib/request_log_analyzer/tracker/frequency.rb spec/fixtures/decompression.log.gz spec/fixtures/decompression.log spec/lib/testing_format.rb spec/fixtures/test_order.log spec/fixtures/rails.db lib/request_log_analyzer/output/fixed_width.rb lib/request_log_analyzer/filter/anonymize.rb lib/request_log_analyzer/tracker/timespan.rb lib/request_log_analyzer/database/base.rb lib/request_log_analyzer/aggregator.rb spec/unit/request_spec.rb lib/cli/progressbar.rb lib/request_log_analyzer/mailer.rb README.rdoc lib/request_log_analyzer/database/warning.rb spec/fixtures/merb.log lib/request_log_analyzer/tracker/hourly_spread.rb .gitignore spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb lib/request_log_analyzer/aggregator/echo.rb spec/unit/mailer_spec.rb spec/unit/controller/log_processor_spec.rb spec/spec_helper.rb lib/request_log_analyzer.rb spec/database.yml Rakefile lib/request_log_analyzer/database/connection.rb spec/unit/filter/filter_spec.rb spec/fixtures/test_language_combined.log lib/request_log_analyzer/aggregator/database_inserter.rb lib/request_log_analyzer/aggregator/summarizer.rb lib/request_log_analyzer/file_format/rack.rb lib/request_log_analyzer/database/source.rb lib/request_log_analyzer/file_format/rails.rb spec/fixtures/decompression.tar.gz spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb lib/request_log_analyzer/filter/timespan.rb lib/request_log_analyzer/source/log_parser.rb spec/fixtures/decompression.tgz spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/fixtures/header_and_footer.log lib/cli/tools.rb lib/request_log_analyzer/file_format/merb.rb spec/fixtures/multiple_files_1.log spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb lib/request_log_analyzer/source.rb lib/request_log_analyzer/request.rb lib/cli/database_console.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/lib/mocks.rb spec/lib/helpers.rb spec/fixtures/rails_1x.log lib/cli/database_console_init.rb lib/request_log_analyzer/output.rb lib/request_log_analyzer/file_format/apache.rb spec/fixtures/decompression.log.zip spec/unit/source/log_parser_spec.rb spec/fixtures/test_file_format.log tasks/github-gem.rake spec/unit/database/database_spec.rb lib/request_log_analyzer/tracker/duration.rb lib/request_log_analyzer/tracker/traffic.rb lib/request_log_analyzer/file_format.rb spec/unit/aggregator/summarizer_spec.rb spec/fixtures/syslog_1x.log spec/fixtures/rails_22.log lib/request_log_analyzer/database/request.rb spec/fixtures/multiple_files_2.log LICENSE lib/request_log_analyzer/source/database_loader.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb lib/cli/command_line_arguments.rb)
+  s.test_files = %w(spec/unit/filter/anonymize_filter_spec.rb spec/unit/file_format/file_format_api_spec.rb spec/unit/file_format/apache_format_spec.rb spec/integration/command_line_usage_spec.rb spec/unit/filter/timespan_filter_spec.rb spec/unit/aggregator/database_inserter_spec.rb spec/unit/request_spec.rb spec/unit/tracker/tracker_api_spec.rb spec/unit/tracker/duration_tracker_spec.rb spec/unit/file_format/amazon_s3_format_spec.rb spec/unit/mailer_spec.rb spec/unit/controller/log_processor_spec.rb spec/unit/filter/filter_spec.rb spec/unit/tracker/traffic_tracker_spec.rb spec/unit/filter/field_filter_spec.rb spec/unit/database/base_class_spec.rb spec/unit/tracker/timespan_tracker_spec.rb spec/unit/tracker/hourly_spread_spec.rb spec/unit/file_format/merb_format_spec.rb spec/unit/file_format/line_definition_spec.rb spec/unit/database/connection_spec.rb spec/unit/controller/controller_spec.rb spec/unit/source/log_parser_spec.rb spec/unit/database/database_spec.rb spec/unit/aggregator/summarizer_spec.rb spec/unit/tracker/frequency_tracker_spec.rb spec/unit/file_format/rails_format_spec.rb)
 end

data/spec/database.yml

@@ -1,3 +1,9 @@
+# This file determines what databases are used to test the database
+# functionality of request-log-analyzer. By default, only an SQLite3
+# memory database is enabled.
+#
+# The syntax of this file is exactly the same as Rails's database.yml.
+
 sqlite3:
   adapter: "sqlite3"
   database: ":memory:"

data/spec/fixtures/rails_22.log

@@ -9,4 +9,4 @@ Rendered shared/_analytics (0.2ms)
 Rendered layouts/_actions (0.6ms)
 Rendered layouts/_menu (2.2ms)
 Rendered layouts/_tabbar (0.5ms)
-Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.coml/demo]
+Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.com/demo]

data/spec/integration/command_line_usage_spec.rb

@@ -15,6 +15,12 @@ describe RequestLogAnalyzer, 'running from command line' do
     output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true
   end
 
+  it "should find 2 requests when parsing a compressed file" do
+    output = run("#{log_fixture(:decompression, :tgz)}")
+    output.any? { |line| /^Parsed requests\:\s*2\s/ =~ line }.should be_true
+  end
+
+
   it "should skip 1 requests with a --select option" do
     output = run("#{log_fixture(:rails_1x)} --select controller PeopleController")
     output.any? { |line| /^Skipped requests\:\s*1\s/ =~ line }.should be_true
@@ -70,7 +76,7 @@ describe RequestLogAnalyzer, 'running from command line' do
     File.exist?(temp_output_file(:dump)).should be_true
     YAML::load(File.read(temp_output_file(:dump))).should have_at_least(1).item
   end
-  
+
   it "should parse 4 requests from the standard input" do
     output = run("- < #{log_fixture(:rails_1x)}")
     output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should be_true

data/spec/lib/helpers.rb

@@ -1,10 +1,10 @@
 module RequestLogAnalyzer::Spec::Helpers
-  
+
   # Create or return a new TestingFormat
   def testing_format
     @testing_format ||= TestingFormat.create
   end
-  
+
   # Load a log file from the fixture folder
   def log_fixture(name, extention = "log")
     File.dirname(__FILE__) + "/../fixtures/#{name}.#{extention}"
@@ -15,7 +15,7 @@ module RequestLogAnalyzer::Spec::Helpers
     if fields.kind_of?(Array)
       format.request(*fields)
     else
-      format.request(fields)      
+      format.request(fields)
     end
   end
 
@@ -24,7 +24,7 @@ module RequestLogAnalyzer::Spec::Helpers
   def run(arguments)
     binary = "#{File.dirname(__FILE__)}/../../bin/request-log-analyzer"
     arguments = arguments.join(' ') if arguments.kind_of?(Array)
-    
+
     output = []
     IO.popen("#{binary} #{arguments}") do |pipe|
       output = pipe.readlines
@@ -32,16 +32,16 @@ module RequestLogAnalyzer::Spec::Helpers
     $?.exitstatus.should == 0
     output
   end
-  
+
   # Cleanup all temporary files generated by specs
   def cleanup_temp_files!
     Dir["#{File.dirname(__FILE__)}/../../tmp/spec.*tmp"].each do |file|
       File.unlink(file)
     end
   end
-  
+
   # Return a filename that can be used as temporary file in specs
   def temp_output_file(file_type)
-    "#{File.dirname(__FILE__)}/../../tmp/spec.#{file_type}.tmp"    
+    "#{File.dirname(__FILE__)}/../../tmp/spec.#{file_type}.tmp"
   end
 end

data/spec/lib/macros.rb

@@ -1,16 +1,16 @@
 module RequestLogAnalyzer::Spec::Macros
-  
+
   def test_databases
     require 'yaml'
     hash = YAML.load(File.read("#{File.dirname(__FILE__)}/../database.yml"))
     hash.inject({}) { |res, (name, h)| res[name] = h.map { |(k,v)| "#{k}=#{v}" }.join(';'); res  }
   end
-  
+
   # Create or return a new TestingFormat
   def testing_format
     @testing_format ||= TestingFormat.create
   end
-  
+
   def default_orm_class_names
     ['Warning', 'Request', 'Source']
   end