request-log-analyzer 1.3.4 → 1.4.0

data/lib/request_log_analyzer/aggregator/summarizer.rb

@@ -1,9 +1,9 @@
 module RequestLogAnalyzer::Aggregator
 
   class Summarizer < Base
-    
+
     class Definer
-      
+
       attr_reader :trackers
 
       # Initialize tracker array
@@ -16,17 +16,17 @@ module RequestLogAnalyzer::Aggregator
       def initialize_copy(other)
         @trackers = other.trackers.dup
       end
-      
+
       # Drop all trackers
       def reset!
         @trackers = []
       end
-      
+
       # Include missing trackers through method missing.
       def method_missing(tracker_method, *args)
         track(tracker_method, *args)
       end
-      
+
       # Track the frequency of a specific category
       # <tt>category_field</tt> Field to track
       # <tt>options</tt> options are passed to new frequency tracker
@@ -37,7 +37,7 @@ module RequestLogAnalyzer::Aggregator
           track(:frequency, category_field.merge(options))
         end
       end
-      
+
       # Track the duration of a specific category
       # <tt>duration_field</tt> Field to track
       # <tt>options</tt> options are passed to new frequency tracker
@@ -45,10 +45,10 @@ module RequestLogAnalyzer::Aggregator
         if duration_field.kind_of?(Symbol)
           track(:duration, options.merge(:duration => duration_field))
         elsif duration_field.kind_of?(Hash)
-          track(:duration, duration_field.merge(options))        
+          track(:duration, duration_field.merge(options))
         end
-      end      
-      
+      end
+
       # Helper function to initialize a tracker and add it to the tracker array.
       # <tt>tracker_class</tt> The class to include
       # <tt>optiont</tt> The options to pass to the trackers.
@@ -57,10 +57,10 @@ module RequestLogAnalyzer::Aggregator
         @trackers << tracker_klass.new(options)
       end
     end
-    
+
     attr_reader :trackers
     attr_reader :warnings_encountered
-    
+
     # Initialize summarizer.
     # Generate trackers from speciefied source.file_format.report_trackers and set them up
     def initialize(source, options = {})
@@ -69,16 +69,16 @@ module RequestLogAnalyzer::Aggregator
       @trackers = source.file_format.report_trackers
       setup
     end
-    
+
     def setup
     end
-    
+
     # Call prepare on all trackers.
     def prepare
       raise "No trackers set up in Summarizer!" if @trackers.nil? || @trackers.empty?
       @trackers.each { |tracker| tracker.prepare }
     end
-    
+
     # Pass all requests to trackers and let them update if necessary.
     # <tt>request</tt> The request to pass.
     def aggregate(request)
@@ -86,19 +86,19 @@ module RequestLogAnalyzer::Aggregator
         tracker.update(request) if tracker.should_update?(request)
       end
     end
-    
+
     # Call finalize on all trackers. Saves a YAML dump if this is set in  the options.
     def finalize
       @trackers.each { |tracker| tracker.finalize }
       save_results_dump(options[:dump]) if options[:dump]
     end
-    
+
     # Saves the results of all the trackers in YAML format to a file.
     # <tt>filename</tt> The file to store the YAML dump in.
     def save_results_dump(filename)
       File.open(filename, 'w') { |file| file.write(to_yaml) }
     end
-    
+
     # Exports all the tracker results to YAML. It will call the to_yaml_object method
     # for every tracker and combines these into a single YAML export.
     def to_yaml
@@ -108,7 +108,7 @@ module RequestLogAnalyzer::Aggregator
       end
       YAML::dump(trackers_export)
     end
-       
+
     # Call report on all trackers.
     # <tt>output</tt> RequestLogAnalyzer::Output object to output to
     def report(output)
@@ -121,13 +121,13 @@ module RequestLogAnalyzer::Aggregator
       end
       report_footer(output)
     end
-    
+
     # Generate report header.
     # <tt>output</tt> RequestLogAnalyzer::Output object to output to
     def report_header(output)
       output.title("Request summary")
-  
-      output.with_style(:cell_separator => false) do 
+
+      output.with_style(:cell_separator => false) do
         output.table({:width => 20}, {:font => :bold}) do |rows|
           rows << ['Parsed lines:',    source.parsed_lines]
           rows << ['Skipped lines:',   source.skipped_lines]
@@ -138,13 +138,13 @@ module RequestLogAnalyzer::Aggregator
       end
       output << "\n"
     end
-    
+
     # Generate report footer.
     # <tt>output</tt> RequestLogAnalyzer::Output object to output to
     def report_footer(output)
       if has_log_ordering_warnings?
         output.title("Parse warnings")
-        
+
         output.puts "Parseable lines were ancountered without a header line before it. It"
         output.puts "could be that logging is not setup correctly for your application."
         output.puts "Visit this website for logging configuration tips:"
@@ -152,17 +152,17 @@ module RequestLogAnalyzer::Aggregator
         output.puts
       end
     end
-    
+
     # Returns true if there were any warnings generated by the trackers
     def has_warnings?
       @warnings_encountered.inject(0) { |result, (key, value)| result += value } > 0
     end
-    
+
     # Returns true if there were any log ordering warnings
     def has_log_ordering_warnings?
       @warnings_encountered[:no_current_request] && @warnings_encountered[:no_current_request] > 0
     end
-    
+
     # Store an encountered warning
     # <tt>type</tt> Type of warning
     # <tt>message</tt> Warning message

data/lib/request_log_analyzer/aggregator.rb

@@ -1,50 +1,46 @@
 module RequestLogAnalyzer::Aggregator
-  
+
   def self.const_missing(const)
     RequestLogAnalyzer::load_default_class_file(self, const)
   end
-  
+
   # The base class of an aggregator. This class provides the interface to which
   # every aggregator should comply (by simply subclassing this class).
   class Base
-    
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    attr_reader :options
-    attr_reader :source
+
+    attr_reader :options, :source
 
     # Intializes a new RequestLogAnalyzer::Aggregator::Base instance
     # It will include the specific file format module.
     def initialize(source, options = {})
       @source = source
-      self.register_file_format(source.file_format)
       @options = options
     end
 
-    # The prepare function is called just before parsing starts. This function 
+    # The prepare function is called just before parsing starts. This function
     # can be used to initialie variables, etc.
     def prepare
     end
-    
+
     # The aggregate function is called for every request.
     # Implement the aggregating functionality in this method
     def aggregate(request)
     end
-    
+
     # The finalize function is called after all sources are parsed and no more
     # requests will be passed to the aggregator
     def finalize
     end
-    
+
     # The warning method is called if the parser eits a warning.
     def warning(type, message, lineno)
-    end    
-    
+    end
+
     # The report function is called at the end. Implement any result reporting
     # in this function.
     def report(output)
     end
-    
+
     # The source_change function gets called when handling a source is started or finished.
     def source_change(change, filename)
     end

data/lib/request_log_analyzer/controller.rb

@@ -1,5 +1,5 @@
 module RequestLogAnalyzer
-  
+
   # The RequestLogAnalyzer::Controller class creates a LogParser instance for the
   # requested file format, and connect it with sources and aggregators.
   #
@@ -17,49 +17,44 @@ module RequestLogAnalyzer
   # from several logrotated log files.
   class Controller
 
-    include RequestLogAnalyzer::FileFormat::Awareness
-    
-    attr_reader :aggregators
-    attr_reader :filters
-    attr_reader :log_parser
-    attr_reader :source
-    attr_reader :output
-    attr_reader :options
+    attr_reader :source, :filters, :aggregators, :output, :options
 
     # Builds a RequestLogAnalyzer::Controller given parsed command line arguments
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
     # <rr>report_with</tt> Width of the report. Defaults to 80.
-    def self.build(arguments)
-      options = { }
-
-      # Database command line options
-      options[:database]       = arguments[:database] if arguments[:database]
+    def self.build_from_arguments(arguments)
+      
+      options = {}
+      
+      # Copy fields
+      options[:database]       = arguments[:database]
       options[:reset_database] = arguments[:reset_database]
-      options[:debug]    = arguments[:debug]
-      options[:dump]     = arguments[:dump]
+      options[:debug]          = arguments[:debug]
+      options[:dump]           = arguments[:dump]
       options[:parse_strategy] = arguments[:parse_strategy]
       options[:no_progress]    = arguments[:no_progress]
+      options[:format]         = arguments[:format]
+      options[:output]         = arguments[:output]
+      options[:file]           = arguments[:file]
+      options[:format]         = arguments[:format]
+      options[:after]          = arguments[:after]
+      options[:before]         = arguments[:before]
+      options[:reject]         = arguments[:reject]
+      options[:select]         = arguments[:select]
+      options[:boring]         = arguments[:boring]
+      options[:aggregator]     = arguments[:aggregator]
+      options[:report_width]   = arguments[:report_width]
+      options[:report_sort]    = arguments[:report_sort]
+      options[:report_amount]  = arguments[:report_amount]
       
-      output_class = RequestLogAnalyzer::Output::const_get(arguments[:output])
-      if arguments[:file]
-        output_file = File.new(arguments[:file], "w+")
-        options[:output] = output_class.new(output_file, :width => 80, :color => false, :characters => :ascii)
-      elsif arguments[:mail]
-        output_mail = RequestLogAnalyzer::Mailer.new(arguments[:mail])
-        options[:output] = output_class.new(output_mail, :width => 80, :color => false, :characters => :ascii)        
-      else
-        options[:output] = output_class.new(STDOUT, :width => arguments[:report_width].to_i, 
-            :color => !arguments[:boring], :characters => (arguments[:boring] ? :ascii : :utf))
+      # Apache format workaround
+      if arguments[:rails_format]
+        options[:format] = {:rails => arguments[:rails_format]}
+      elsif arguments[:apache_format]
+        options[:format] = {:apache => arguments[:apache_format]}
       end
-                
-      # Create the controller with the correct file format
-      file_format = if arguments[:apache_format]
-          RequestLogAnalyzer::FileFormat.load(:apache, arguments[:apache_format])
-        else 
-          RequestLogAnalyzer::FileFormat.load(arguments[:format])
-        end
-
-      # register sources
+      
+      # Register sources
       if arguments.parameters.length == 1
         file = arguments.parameters[0]
         if file == '-' || file == 'STDIN'
@@ -74,49 +69,128 @@ module RequestLogAnalyzer
         options.store(:source_files, arguments.parameters)
       end
       
-      controller = Controller.new(RequestLogAnalyzer::Source::LogParser.new(file_format, options), options)
-      #controller = Controller.new(RequestLogAnalyzer::Source::DatabaseLoader.new(file_format, options), options)
+      build(options)
+    end
+
+    # Build a new controller using parameters (Base for new API)
+    # <tt>source</tt> The source file
+    # Options are passd on to the LogParser.
+    #
+    # Options
+    # * <tt>:database</tt> Database file
+    # * <tt>:reset_database</tt> 
+    # * <tt>:debug</tt> Enables echo aggregator.
+    # * <tt>:dump</tt> 
+    # * <tt>:parse_strategy</tt> 
+    # * <tt>:no_progress</tt> 
+    # * <tt>:output</tt> :fixed_width, :html or Output class. Defaults to fixed width.
+    # * <tt>:file</tt> Filestring or File or StringIO
+    # * <tt>:format</tt> :rails, {:apache => 'FORMATSTRING'}, :merb, etcetera or Format Class. Defaults to :rails.
+    # * <tt>:source_files</tt> File or STDIN
+    # * <tt>:after</tt> Drop all requests after this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
+    # * <tt>:before</tt> Drop all requests before this date (Date, DateTime, Time, or a String in "YYYY-MM-DD hh:mm:ss" format)
+    # * <tt>:reject</tt> Reject specific {:field => :value} combination. Expects single hash.
+    # * <tt>:select</tt> Select specific {:field => :value} combination. Expects single hash.
+    # * <tt>:aggregator</tt> Array of aggregators (ATM: STRINGS OR SYMBOLS ONLY!). Defaults to [:summarizer
+    # * <tt>:boring</tt> Do not show color on STDOUT. Defaults to False.
+    # * <tt>:report_width</tt> Width or reports in characters. Defaults to 80.
+    #
+    # TODO:
+    #   Check if defaults work (Aggregator defaults seem wrong).
+    #   Refactor :database => options[:database], :dump => options[:dump] away from contoller intialization.
+    def self.build(options)
+      # Defaults
+      options[:output]        ||= :fixed_width
+      options[:format]        ||= :rails
+      options[:aggregator]    ||= [:summarizer]
+      options[:report_width]  ||= 80
+      options[:report_amount] ||= 20
+      options[:report_sort]   ||= 'sum,mean'
+      options[:boring]        ||= false
+      
+      # Set the output class
+      output_args   = {}
+      output_object = nil
+      if options[:output].is_a? Class
+        output_class = options[:output]
+      else
+        output_class = RequestLogAnalyzer::Output::const_get(options[:output])
+      end
+      
+      output_sort   = options[:report_sort].split(',').map { |s| s.to_sym }
+      output_amount = options[:report_amount] == 'all' ? :all : options[:report_amount].to_i
+      
+      if options[:file]
+        output_object = %w[File StringIO].include?(options[:file].class.name) ? options[:file] : File.new(options[:file], "w+")
+        output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount }
+      elsif options[:mail]
+        output_object = RequestLogAnalyzer::Mailer.new(arguments[:mail])
+        output_args   = {:width => 80, :color => false, :characters => :ascii, :sort => output_sort, :amount => output_amount  }
+      else
+        output_object = STDOUT
+        output_args   = {:width => options[:report_width].to_i, :color => !options[:boring],
+                        :characters => (options[:boring] ? :ascii : :utf), :sort => output_sort, :amount => output_amount }
+      end
+      
+      output_instance = output_class.new(output_object, output_args)
+      
+      # Create the controller with the correct file format
+      if options[:format].kind_of?(Hash)
+        file_format = RequestLogAnalyzer::FileFormat.load(options[:format].keys[0], options[:format].values[0])
+      else
+        file_format = RequestLogAnalyzer::FileFormat.load(options[:format])
+      end
       
+      # Kickstart the controller
+      controller = Controller.new(  RequestLogAnalyzer::Source::LogParser.new(file_format, :source_files => options[:source_files]),
+                                    { :output => output_instance,
+                                      :database => options[:database],                # FUGLY!
+                                      :dump => options[:dump], 
+                                      :reset_database => options[:reset_database]})
+
       # register filters
-      if arguments[:after] || arguments[:before]
+      if options[:after] || options[:before]
         filter_options = {}
-        filter_options[:after]  = DateTime.parse(arguments[:after])  
-        filter_options[:before] = DateTime.parse(arguments[:before]) if arguments[:before]
+        [:after, :before].each do |filter|
+          case options[filter]
+          when Date, DateTime, Time
+            filter_options[filter] = options[filter]
+          when String
+            filter_options[filter] = DateTime.parse(options[filter])
+          end
+        end
         controller.add_filter(:timespan, filter_options)
       end
-      
-      arguments[:reject].each do |(field, value)|
-        controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
+
+      if options[:reject]
+        options[:reject].each do |(field, value)|
+          controller.add_filter(:field, :mode => :reject, :field => field, :value => value)
+        end
       end
-      
-      arguments[:select].each do |(field, value)|
-        controller.add_filter(:field, :mode => :select, :field => field, :value => value)
+
+      if options[:reject]
+        options[:select].each do |(field, value)|
+          controller.add_filter(:field, :mode => :select, :field => field, :value => value)
+        end
       end
 
       # register aggregators
-      arguments[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
+      options[:aggregator].each { |agg| controller.add_aggregator(agg.to_sym) }
+      controller.add_aggregator(:summarizer)          if options[:aggregator].empty?
+      controller.add_aggregator(:echo)                if options[:debug]
+      controller.add_aggregator(:database_inserter)   if options[:database] && !options[:aggregator].include?('database')
 
-      # register the database 
-      controller.add_aggregator(:summarizer)          if arguments[:aggregator].empty?
-      controller.add_aggregator(:database_inserter)   if arguments[:database] && !arguments[:aggregator].include?('database')
-
-      # register the echo aggregator in debug mode
-      controller.add_aggregator(:echo) if arguments[:debug]
-      
       file_format.setup_environment(controller)
-          
       return controller
-    end
+    end  
 
     # Builds a new Controller for the given log file format.
     # <tt>format</tt> Logfile format. Defaults to :rails
     # Options are passd on to the LogParser.
-    # * <tt>:aggregator</tt> Aggregator array.
     # * <tt>:database</tt> Database the controller should use.
-    # * <tt>:echo</tt> Output debug information.
-    # * <tt>:silent</tt> Do not output any warnings.
-    # * <tt>:colorize</tt> Colorize output
+    # * <tt>:dump</tt> Yaml Dump the contrller should use.
     # * <tt>:output</tt> All report outputs get << through this output.
+    # * <tt>:no_progress</tt> No progress bar
     def initialize(source, options = {})
 
       @source      = source
@@ -124,20 +198,17 @@ module RequestLogAnalyzer
       @aggregators = []
       @filters     = []
       @output      = options[:output]
+      @interrupted = false
       
-      # Requester format through RequestLogAnalyzer::FileFormat and construct the parser
-      register_file_format(@source.file_format) 
-      
-      # Pass all warnings to every aggregator so they can do something useful with them.
-      @source.warning = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } } if @source
+      # Register the request format for this session after checking its validity
+      raise "Invalid file format!" unless @source.file_format.valid?
 
-      # Handle progress messagess
-      @source.progress = lambda { |message, value| handle_progress(message, value) } if @source && !options[:no_progress]
-      
-      # Handle source change messages
-      @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) } if @source
+      # Install event handlers for wrnings, progress updates and source changes
+      @source.warning        = lambda { |type, message, lineno|  @aggregators.each { |agg| agg.warning(type, message, lineno) } }
+      @source.progress       = lambda { |message, value| handle_progress(message, value) } unless options[:no_progress]
+      @source.source_changes = lambda { |change, filename| handle_source_change(change, filename) }
     end
-    
+
     # Progress function.
     # Expects :started with file, :progress with current line and :finished or :interrupted when done.
     # <tt>message</tt> Current state (:started, :finished, :interupted or :progress).
@@ -158,46 +229,46 @@ module RequestLogAnalyzer
         @progress_bar.set(value)
       end
     end
-    
+
     # Source change handler
     def handle_source_change(change, filename)
       @aggregators.each { |agg| agg.source_change(change, File.expand_path(filename, Dir.pwd)) }
     end
-    
-    # Adds an aggregator to the controller. The aggregator will be called for every request 
+
+    # Adds an aggregator to the controller. The aggregator will be called for every request
     # that is parsed from the provided sources (see add_source)
-    def add_aggregator(agg)      
+    def add_aggregator(agg)
       agg = RequestLogAnalyzer::Aggregator.const_get(RequestLogAnalyzer::to_camelcase(agg)) if agg.kind_of?(Symbol)
       @aggregators << agg.new(@source, @options)
     end
-    
+
     alias :>> :add_aggregator
-    
+
     # Adds a request filter to the controller.
     def add_filter(filter, filter_options = {})
       filter = RequestLogAnalyzer::Filter.const_get(RequestLogAnalyzer::to_camelcase(filter)) if filter.kind_of?(Symbol)
-      @filters << filter.new(file_format, @options.merge(filter_options))
+      @filters << filter.new(source.file_format, @options.merge(filter_options))
     end
-    
+
     # Push a request through the entire filterchain (@filters).
     # <tt>request</tt> The request to filter.
     # Returns the filtered request or nil.
     def filter_request(request)
-      @filters.each do |filter| 
+      @filters.each do |filter|
         request = filter.filter(request)
         return nil if request.nil?
       end
       return request
     end
-    
+
     # Push a request to all the aggregators (@aggregators).
-    # <tt>request</tt> The request to push to the aggregators.    
+    # <tt>request</tt> The request to push to the aggregators.
     def aggregate_request(request)
       return false unless request
       @aggregators.each { |agg| agg.aggregate(request) }
       return true
     end
-    
+
     # Runs RequestLogAnalyzer
     # 1. Call prepare on every aggregator
     # 2. Generate requests from source object
@@ -207,10 +278,12 @@ module RequestLogAnalyzer
     # 5. Call report on every aggregator
     # 6. Finalize Source
     def run!
-      
+
+      # @aggregators.each{|agg| p agg}
+
       @aggregators.each { |agg| agg.prepare }
       install_signal_handlers
-      
+
       @source.each_request do |request|
         break if @interrupted
         aggregate_request(filter_request(request))
@@ -221,9 +294,9 @@ module RequestLogAnalyzer
       @output.header
       @aggregators.each { |agg| agg.report(@output) }
       @output.footer
-            
+
       @source.finalize
-      
+
       if @output.io.kind_of?(File)
         puts
         puts "Report written to: " + File.expand_path(@output.io.path)
@@ -235,7 +308,7 @@ module RequestLogAnalyzer
         @output.io.mail
       end
     end
-    
+
     def install_signal_handlers
       Signal.trap("INT") do
         handle_progress(:interrupted)
@@ -243,6 +316,6 @@ module RequestLogAnalyzer
         @interrupted = true
       end
     end
-    
+
   end
 end