request-log-analyzer 1.7.0 → 1.10.0

data/lib/request_log_analyzer/file_format/haproxy.rb

@@ -0,0 +1,138 @@
+module RequestLogAnalyzer::FileFormat
+
+  class Haproxy < RequestLogAnalyzer::FileFormat::Base
+
+    extend CommonRegularExpressions
+
+    # Define line types
+    line_definition :haproxy do |line|
+      line.header = true
+      line.footer = true
+
+      line.regexp = %r{
+        (#{ip_address}):\d+\s # client_ip ':' client_port
+        \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})\.\d{3}\]\s # '[' accept_date ']'
+        (\S+)\s # frontend_name
+        (\S+)\/(\S+)\s # backend_name '/' server_name
+        (\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        (\d+)\s # status_code
+        \+?(\d+)\s # bytes_read
+        (\S+)\s # captured_request_cookie
+        (\S+)\s # captured_response_cookie
+        (\w|-)(\w|-)(\w|-)(\w|-)\s # termination_state
+        (\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
+        (\d+)\/(\d+)\s # srv_queue '/' backend_queue
+        (\S*)\s? # captured_request_headers
+        (\S*)\s? # captured_response_headers
+        "([^"]*)" # '"' http_request '"'
+      }x
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:accept_date).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:backend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tw).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:actconn).as(:integer)
+      line.capture(:feconn).as(:integer)
+      line.capture(:beconn).as(:integer)
+      line.capture(:srv_conn).as(:integer)
+      line.capture(:retries).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
+    # Define the summary report
+    report do |analyze|
+      analyze.hourly_spread :field => :accept_date
+
+      analyze.frequency :client_ip,
+        :title => "Hits per IP"
+
+      analyze.frequency :frontend_name,
+        :title => "Hits per frontend service"
+
+      analyze.frequency :backend_name,
+        :title => "Hits per backend service"
+
+      analyze.frequency :server_name,
+        :title => "Hits per backend server"
+
+      analyze.frequency :status_code,
+        :title => "HTTP response code frequency"
+
+      analyze.frequency :http_request,
+        :title => "Most popular requests"
+
+      analyze.frequency :http_request,
+        :title => "Most frequent HTTP 40x errors",
+        :category => lambda { |r| "#{r[:http_request]}"},
+        :if => lambda { |r| r[:status_code] >= 400 and r[:status_code] <= 417 }
+
+      analyze.frequency :http_request,
+        :title => "Most frequent HTTP 50x errors",
+        :category => lambda { |r| "#{r[:http_request]}"},
+        :if => lambda { |r| r[:status_code] >= 500 and r[:status_code] <= 505 }
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per frontend service",
+        :category => lambda { |r| "#{r[:frontend_name]}"}
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per backend service",
+        :category => lambda { |r| "#{r[:backend_name]}"}
+
+      analyze.traffic :bytes_read,
+        :title => "Traffic per backend server",
+        :category => lambda { |r| "#{r[:server_name]}"}
+
+      analyze.duration :tr,
+        :title => "Time waiting for backend response",
+        :category => lambda { |r| "#{r[:http_request]}"}
+
+      analyze.duration :tt,
+        :title => "Total time spent on request",
+        :category => lambda { |r| "#{r[:http_request]}"}
+    end
+
+    # Define a custom Request class for the HAProxy file format to speed up
+    # timestamp handling. Shamelessly copied from apache.rb
+    class Request < RequestLogAnalyzer::Request
+
+      MONTHS = {'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06',
+                'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' }
+
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return
+      # a integer to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[7,4]}#{MONTHS[value[3,3]]}#{value[0,2]}#{value[12,2]}#{value[15,2]}#{value[18,2]}".to_i
+      end
+
+      # Make sure that the strings '-' or '{}' or '' are parsed as a nil value.
+      def convert_nillable_string(value, definition)
+        value =~ /-|\{\}|^$/ ? nil : value
+      end
+
+      # Make sure that -1 is parsed as a nil value.
+      def convert_nillable_duration(value, definition)
+        value == '-1' ? nil : convert_duration(value, definition)
+      end
+
+    end
+  end
+end

data/lib/request_log_analyzer/file_format/oink.rb

@@ -0,0 +1,96 @@
+class RequestLogAnalyzer::FileFormat::Oink < RequestLogAnalyzer::FileFormat::Rails
+  # capture the PID
+  HODEL3000_PROCESSING = /\[(\d+)\]: Processing ((?:\w+::)*\w+)#(\w+)(?: to (\w+))? \(for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})\) \[([A-Z]+)\]/
+  
+  # TODO: fix me!
+  line_definition :processing do |line|
+    line.header = true
+    line.regexp = Regexp.union(LINE_DEFINITIONS[:processing].regexp,HODEL3000_PROCESSING)
+    line.captures = [   { :name => :controller, :type  => :string }, # Default Rails Processing
+                        { :name => :action,     :type  => :string },
+                        { :name => :format,     :type  => :string, :default => 'html' },
+                        { :name => :ip,         :type  => :string },
+                        { :name => :timestamp,  :type  => :timestamp },
+                        { :name => :method,     :type  => :string },
+                        { :name => :pid,        :type  => :integer }, # Hodel 3000 Processing w/PID
+                        { :name => :controller, :type  => :string },
+                        { :name => :action,     :type  => :string },
+                        { :name => :format,     :type  => :string, :default => 'html' },
+                        { :name => :ip,         :type  => :string },
+                        { :name => :timestamp,  :type  => :timestamp },
+                        { :name => :method,     :type  => :string }]
+  end
+  
+  line_definition :memory_usage do |line|
+    line.regexp   = /\[(\d+)\]: Memory usage: (\d+)/
+    line.capture(:pid).as(:integer)
+    line.capture(:memory).as(:traffic)
+  end
+
+  line_definition :instance_type_counter do |line|
+    line.regexp = /\[(\d+)\]: Instantiation Breakdown: (.*)$/
+    line.capture(:pid).as(:integer) 
+    line.capture(:instance_counts).as(:pipe_separated_counts)
+  end
+  
+  report(:append) do |analyze|
+    analyze.traffic :memory_diff, :category => REQUEST_CATEGORIZER, :title => "Largest Memory Increases", :line_type => :memory_usage
+  end
+  
+  # Keep a record of PIDs and their memory usage when validating requests.
+  def pids
+    @pids ||= {}
+  end
+
+  class Request < RequestLogAnalyzer::FileFormat::Rails::Request
+   # Overrides the #validate method to handle PID updating.
+   def validate
+     update_pids
+     super
+   end
+   
+   # Accessor for memory information associated with the specified request PID. If no memory exists
+   # for this request's :pid, the memory tracking is initialized.
+   def pid_memory
+     file_format.pids[self[:pid]] ||= { :last_memory_reading => -1, :current_memory_reading => -1 }
+   end
+    
+   # Calculates :memory_diff for each request based on the last completed request that was not a failure.
+   def update_pids
+     # memory isn't recorded with exceptions. need to set #last_memory_reading+ to -1 as
+     # the memory used could have changed. for the next request the memory change will not be recorded.
+     #
+     # NOTE - the failure regex was not matching with a Rails Development log file.
+     if has_line_type?(:failure) and processing = has_line_type?(:processing)
+       pid_memory[:last_memory_reading] = -1
+     elsif mem_line = has_line_type?(:memory_usage)
+        memory_reading = mem_line[:memory]
+        pid_memory[:current_memory_reading] = memory_reading
+        # calcuate the change in memory
+        unless pid_memory[:current_memory_reading] == -1 || pid_memory[:last_memory_reading] == -1
+          # logged as kB, need to convert to bytes for the :traffic Tracker
+          memory_diff = (pid_memory[:current_memory_reading] - pid_memory[:last_memory_reading])*1024
+          if memory_diff > 0
+            self.attributes[:memory_diff] = memory_diff
+          end # if memory_diff > 0
+        end # unless
+        
+        pid_memory[:last_memory_reading] = pid_memory[:current_memory_reading]
+        pid_memory[:current_memory_reading] = -1
+      end # if mem_line
+      return true
+   end # def update_pids
+
+  def convert_pipe_separated_counts(value, capture_definition)
+    count_strings = value.split(' | ')
+    count_arrays = count_strings.map do |count_string|
+      if count_string =~ /^(\w+): (\d+)/
+        [$1, $2.to_i]
+      end
+    end
+
+    Hash[count_arrays]
+  end
+  
+  end # class Request
+end

data/lib/request_log_analyzer/file_format/postgresql.rb

@@ -1,24 +1,25 @@
 module RequestLogAnalyzer::FileFormat
 
+  # PostgresQL spec 8.3.7
   class Postgresql < Base
 
     extend CommonRegularExpressions
     
     line_definition :query do |line|
       line.header = true
-      line.teaser = /LOG\:  query\:/
-      line.regexp = /(#{timestamp('%y-%m-%d %k:%M:%S')})\ LOG:  query:\s+(.*)/
+      line.teaser = /.*LOG\:/
+      line.regexp = /(#{timestamp('%Y-%m-%d %k:%M:%S')})\ \S+ \[\d+\]\:\ \[.*\]\ LOG\:\ \ \d+\:\ duration\: (.*)\ ms\ \ statement:\ (.*)/
 
       line.capture(:timestamp).as(:timestamp)
+      line.capture(:query_time).as(:duration, :unit => :sec)
       line.capture(:query_fragment)
     end
       
-    line_definition :duration do |line|
+    line_definition :location do |line|
       line.footer = true
-      line.teaser = /duration:/
-      line.regexp = /#{timestamp('%y-%m-%d %k:%M:%S')}\ LOG\:  duration\: (.*)(\ )sec/
+      line.teaser = /.*LOCATION:/
+      line.regexp = /.*(\ )LOCATION:/
 
-      line.capture(:query_time).as(:duration, :unit => :sec)
       line.capture(:query).as(:sql) # Hack to gather up fragments
     end
     

data/lib/request_log_analyzer/file_format/rails.rb

@@ -25,9 +25,9 @@ module RequestLogAnalyzer::FileFormat
       lines.each do |line|
         line = line.to_sym
         if LINE_COLLECTIONS.has_key?(line)
-          LINE_COLLECTIONS[line].each { |l| definitions_hash[l] = LINE_DEFINITIONS[l] }
+          LINE_COLLECTIONS[line].each { |l| definitions_hash[l] ||= LINE_DEFINITIONS[l] }
         elsif LINE_DEFINITIONS.has_key?(line)
-          definitions_hash[line] = LINE_DEFINITIONS[line]
+          definitions_hash[line] ||= LINE_DEFINITIONS[line]
         else
           raise "Unrecognized Rails log line name: #{line.inspect}!"
         end
@@ -82,8 +82,8 @@ module RequestLogAnalyzer::FileFormat
 
     # Rails > 2.1 completed line example
     # Completed in 614ms (View: 120, DB: 31) | 200 OK [http://floorplanner.local/demo]
-    RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+), )?DB: (\d+)\) \| (\d\d\d).+\[(http.+)\]/
-
+    RAILS_22_COMPLETED = /Completed in (\d+)ms \((?:View: (\d+))?,?(?:.?DB: (\d+))?\)? \| (\d{3}).+\[(http.+)\]/
+                        
     # A hash of definitions for all common lines in Rails logs.
     LINE_DEFINITIONS = {
       :processing => RequestLogAnalyzer::LineDefinition.new(:processing, :header => true,

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -8,11 +8,11 @@ module RequestLogAnalyzer::FileFormat
 
     extend CommonRegularExpressions
 
-    # Started GET "/queries" for 127.0.0.1 at 2010-02-25 16:15:18
+    # beta4: Started GET "/" for 127.0.0.1 at Wed Jul 07 09:13:27 -0700 2010 (different time format)
     line_definition :started do |line|
       line.header = true
       line.teaser = /Started /
-      line.regexp = /Started ([A-Z]+) "([^"]+)" for (#{ip_address}) at (#{timestamp('%Y-%m-%d %H:%M:%S')})/
+      line.regexp = /Started ([A-Z]+) "([^"]+)" for (#{ip_address}) at (#{timestamp('%a %b %d %H:%M:%S %z %Y')}|#{timestamp('%Y-%m-%d %H:%M:%S %z')})/
       
       line.capture(:method)
       line.capture(:path)
@@ -23,21 +23,25 @@ module RequestLogAnalyzer::FileFormat
     # Processing by QueriesController#index as HTML
     line_definition :processing do |line|
       line.teaser = /Processing by /
-      line.regexp = /Processing by (\w+)\#(\w+) as (\w+)/
+      line.regexp = /Processing by ([A-Za-z0-9\-:]+)\#(\w+) as ([\w\/\*]+)/
       
       line.capture(:controller)
       line.capture(:action)
       line.capture(:format)
     end
     
-    # Completed in 9ms (Views: 4.9ms | ActiveRecord: 0.5ms) with 200
+    # Completed 200 OK in 224ms (Views: 200.2ms | ActiveRecord: 3.4ms)
+    # Completed 302 Found in 23ms
+    # Completed   in 189ms
     line_definition :completed do |line|
       line.footer = true
       line.teaser = /Completed /
-      line.regexp = /Completed (\d+) .* in (\d+)ms \([^\)]*\)/
-
+      line.regexp = /Completed (\d+)? .*in (\d+(?:\.\d+)?)ms(?:[^\(]*\(Views: (\d+(?:\.\d+)?)ms .* ActiveRecord: (\d+(?:\.\d+)?)ms.*\))?/
+      
       line.capture(:status).as(:integer)
       line.capture(:duration).as(:duration, :unit => :msec)
+      line.capture(:view).as(:duration, :unit => :msec)
+      line.capture(:db).as(:duration, :unit => :msec)
     end
     
     # ActionView::Template::Error (undefined local variable or method `field' for #<Class>) on line #3 of /Users/willem/Code/warehouse/app/views/queries/execute.csv.erb:
@@ -69,18 +73,37 @@ module RequestLogAnalyzer::FileFormat
       analyze.frequency :status, :title => 'HTTP statuses returned'
       
       analyze.duration :duration, :category => REQUEST_CATEGORIZER, :title => "Request duration",    :line_type => :completed
-      # analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
-      # analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
+      analyze.duration :view,     :category => REQUEST_CATEGORIZER, :title => "View rendering time", :line_type => :completed
+      analyze.duration :db,       :category => REQUEST_CATEGORIZER, :title => "Database time",       :line_type => :completed
       
       analyze.frequency :category => REQUEST_CATEGORIZER, :title => 'Process blockers (> 1 sec duration)',
         :if => lambda { |request| request[:duration] && request[:duration] > 1.0 }
     end
     
     class Request < RequestLogAnalyzer::Request
-      def convert_timestamp(value, defintion)
-        value.gsub(/[^0-9]/, '')[0...14].to_i
+      # Used to handle conversion of abbrev. month name to a digit
+      MONTHS = %w(Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec)
+      
+      def convert_timestamp(value, definition)
+        # the time value can be in 2 formats: 
+        # - 2010-10-26 02:27:15 +0000 (ruby 1.9.2)
+        # - Thu Oct 25 16:15:18 -0800 2010
+        if value =~ /^#{CommonRegularExpressions::TIMESTAMP_PARTS['Y']}/
+          value.gsub!(/\W/,'')
+          value[0..13].to_i
+        else
+          value.gsub!(/\W/,'')
+          time_as_str = value[-4..-1] # year
+          # convert the month to a 2-digit representation
+          month = MONTHS.index(value[3..5])+1
+          month < 10 ? time_as_str << "0#{month}" : time_as_str << month.to_s
+
+          time_as_str << value[6..13] # day of month + time
+          time_as_str.to_i
+        end
+
       end
     end
 
   end
-end
+end

data/lib/request_log_analyzer/file_format/w3c.rb

@@ -0,0 +1,56 @@
+module RequestLogAnalyzer::FileFormat
+
+  # FileFormat for W3C access logs.
+  class W3c < Base
+    
+    extend CommonRegularExpressions
+
+    line_definition :access do |line|
+      line.header = true
+      line.footer = true
+      line.regexp = /^(#{timestamp('%Y-%m-%d %H:%M:%S')}) (#{ip_address}) (.*) (#{ip_address}) (\d+) (\w+) ([\w|\/|.]+) \- (\d+) (\d+) (\d+) (\d+) (.*) (\S+)/
+      
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:remote_ip)
+      line.capture(:username).as(:nillable_string)
+      line.capture(:local_ip)
+      line.capture(:port).as(:integer)
+      line.capture(:method)
+      line.capture(:path).as(:path)
+      line.capture(:http_status).as(:integer)
+      line.capture(:bytes_sent).as(:traffic, :unit => :byte)
+      line.capture(:bytes_received).as(:traffic, :unit => :byte)
+      line.capture(:duration).as(:duration, :unit => :msec)
+      line.capture(:user_agent)
+      line.capture(:referer)
+    end
+
+    report do |analyze|
+      analyze.timespan
+      analyze.hourly_spread
+
+      analyze.frequency :category => :http_method, :title => "HTTP methods"
+      analyze.frequency :category => :http_status, :title => "HTTP statuses"
+      
+      analyze.frequency :category => :path, :title => "Most popular URIs"
+
+      analyze.frequency :category => :user_agent, :title => "User agents"
+      analyze.frequency :category => :referer,    :title => "Referers"
+
+      analyze.duration :duration => :duration,  :category => :path, :title => 'Request duration'
+      analyze.traffic  :traffic => :bytes_sent, :category => :path, :title => 'Traffic out'
+      analyze.traffic  :traffic => :bytes_received, :category => :path, :title => 'Traffic in'
+    end
+    
+    class Request < RequestLogAnalyzer::Request
+      # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer
+      # to speed up parsing.
+      def convert_timestamp(value, definition)
+        "#{value[0,4]}#{value[5,2]}#{value[8,2]}#{value[11,2]}#{value[14,2]}#{value[17,2]}".to_i
+      end
+    end
+    
+
+  end
+  
+end

data/lib/request_log_analyzer/file_format.rb

@@ -1,8 +1,20 @@
 module RequestLogAnalyzer::FileFormat
 
-  def self.const_missing(const) # :nodoc:
-    RequestLogAnalyzer::load_default_class_file(self, const)
-  end
+  autoload :Rails,            'request_log_analyzer/file_format/rails'
+  autoload :Rails3,           'request_log_analyzer/file_format/rails3'
+  autoload :RailsDevelopment, 'request_log_analyzer/file_format/rails_development'
+  autoload :Oink,             'request_log_analyzer/file_format/oink'
+  autoload :Rack,             'request_log_analyzer/file_format/rack'
+  autoload :Merb,             'request_log_analyzer/file_format/merb'
+  autoload :Mysql,            'request_log_analyzer/file_format/mysql'
+  autoload :Postgresql,       'request_log_analyzer/file_format/postgresql'
+  autoload :DelayedJob,       'request_log_analyzer/file_format/delayed_job'
+  autoload :DelayedJob2,      'request_log_analyzer/file_format/delayed_job2'
+  autoload :DelayedJob21,     'request_log_analyzer/file_format/delayed_job21'
+  autoload :Apache,           'request_log_analyzer/file_format/apache'
+  autoload :AmazonS3,         'request_log_analyzer/file_format/amazon_s3'
+  autoload :W3c,              'request_log_analyzer/file_format/w3c'
+  autoload :Haproxy,          'request_log_analyzer/file_format/haproxy'
 
   # Loads a FileFormat::Base subclass instance.
   # You can provide:
@@ -20,10 +32,11 @@ module RequestLogAnalyzer::FileFormat
       # a usable class is provided. Use this format class.
       klass = file_format
 
-    elsif file_format.kind_of?(String) && File.exist?(file_format)
+    elsif file_format.kind_of?(String) && File.exist?(file_format) && File.file?(file_format)
       # load a format from a ruby file
       require file_format
-      const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
+
+      const = RequestLogAnalyzer.to_camelcase(File.basename(file_format, '.rb'))
       if RequestLogAnalyzer::FileFormat.const_defined?(const)
         klass = RequestLogAnalyzer::FileFormat.const_get(const)
       elsif Object.const_defined?(const)
@@ -34,7 +47,7 @@ module RequestLogAnalyzer::FileFormat
 
     else
       # load a provided file format
-      klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer::to_camelcase(file_format))
+      klass = RequestLogAnalyzer::FileFormat.const_get(RequestLogAnalyzer.to_camelcase(file_format))
     end
 
     # check the returned klass to see if it can be used
@@ -46,7 +59,7 @@ module RequestLogAnalyzer::FileFormat
   
   # Returns an array of all FileFormat instances that are shipped with request-log-analyzer by default.
   def self.all_formats
-    @all_formats ||= Dir[File.dirname(__FILE__) + '/file_format/*.rb'].map do |file| 
+    @all_formats ||= Dir[File.expand_path('file_format/*.rb', File.dirname(__FILE__))].map do |file| 
       self.load(File.basename(file, '.rb')) 
     end
   end
@@ -246,11 +259,21 @@ module RequestLogAnalyzer::FileFormat
       request_class.create(self, *hashes)
     end
 
-    # Checks whether the line definitions form a valid language.
-    # A file format should have at least a header and a footer line type
+    # Checks whether the file format is valid so it can be safely used with RLA.
     def valid?
+      valid_line_definitions? && valid_request_class?
+    end
+
+    # Checks whether the line definitions form a valid language.
+    # A file format should have at least a header and a footer line type    
+    def valid_line_definitions?
       line_definitions.any? { |(name, ld)| ld.header } && line_definitions.any? { |(name, ld)| ld.footer }
     end
+    
+    # Checks whether the request class inherits from the base Request class.
+    def valid_request_class?
+      request_class.ancestors.include?(RequestLogAnalyzer::Request)
+    end
 
     # Returns true if this language captures the given symbol in one of its line definitions
     def captures?(name)
@@ -272,4 +295,4 @@ module RequestLogAnalyzer::FileFormat
       return nil
     end
   end
-end
+end

data/lib/request_log_analyzer/filter.rb

@@ -1,10 +1,8 @@
 module RequestLogAnalyzer::Filter
 
-  # Filter class loader using const_missing
-  # This function will automatically load the class file based on the name of the class
-  def self.const_missing(const)
-    RequestLogAnalyzer::load_default_class_file(self, const)
-  end
+  autoload :Field,     'request_log_analyzer/filter/field'
+  autoload :Timespan,  'request_log_analyzer/filter/timespan'
+  autoload :Anonymize, 'request_log_analyzer/filter/anonymize'
 
   # Base filter class used to filter input requests.
   # All filters should interit from this base.

data/lib/request_log_analyzer/output/fancy_html.rb

@@ -10,11 +10,6 @@ module RequestLogAnalyzer::Output
   
   class FancyHTML < HTML
     
-    # Load class files if needed
-    def self.const_missing(const)
-      RequestLogAnalyzer::load_default_class_file(self, const)
-    end
-    
     def report_tracker(tracker)
       case tracker
       when RequestLogAnalyzer::Tracker::HourlySpread then report_hourly_spread(tracker)

data/lib/request_log_analyzer/output.rb

@@ -1,11 +1,10 @@
 # Module for generating output
 module RequestLogAnalyzer::Output
 
-  # Load class files if needed
-  def self.const_missing(const)
-    RequestLogAnalyzer::load_default_class_file(self, const)
-  end
-
+  autoload :FixedWidth, 'request_log_analyzer/output/fixed_width'
+  autoload :HTML,       'request_log_analyzer/output/html'
+  autoload :FancyHTML,  'request_log_analyzer/output/fancy_html'
+  
   # Loads a Output::Base subclass instance.
   def self.load(file_format, *args)
     
@@ -21,7 +20,7 @@ module RequestLogAnalyzer::Output
     elsif file_format.kind_of?(String) && File.exist?(file_format)
       # load a format from a ruby file
       require file_format
-      const = RequestLogAnalyzer::to_camelcase(File.basename(file_format, '.rb'))
+      const = RequestLogAnalyzer.to_camelcase(File.basename(file_format, '.rb'))
       if RequestLogAnalyzer::FileFormat.const_defined?(const)
         klass = RequestLogAnalyzer::Output.const_get(const)
       elsif Object.const_defined?(const)
@@ -32,7 +31,7 @@ module RequestLogAnalyzer::Output
 
     else
       # load a provided file format
-      klass = RequestLogAnalyzer::Output.const_get(RequestLogAnalyzer::to_camelcase(file_format))
+      klass = RequestLogAnalyzer::Output.const_get(RequestLogAnalyzer.to_camelcase(file_format))
     end
 
     # check the returned klass to see if it can be used

data/lib/request_log_analyzer/request.rb

@@ -27,14 +27,28 @@ module RequestLogAnalyzer
       def convert_integer(value, capture_definition); value.to_i; end
       def convert_sym(value, capture_definition);     value.to_sym; end
       def convert_symbol(value, capture_definition);  value.to_sym; end
+      def convert_nillable_string(value, definition); value == '-' ? nil : value ; end
+      
+      # This function can be overridden to rewrite the path for better categorization in the
+      # reports.
+      def convert_path(value, definition)
+        value
+      end
 
       # Converts :eval field, which should evaluate to a hash.
       def convert_eval(value, capture_definition)
-        eval(value).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
-      rescue SyntaxError
+        eval(sanitize_parameters(value)).inject({}) { |h, (k, v)| h[k.to_sym] = v; h}
+			# Wide range of errors possible with wild eval. ATM we choose to crash on this.
+      rescue SyntaxError 
         nil
       end
 
+      # Removes certain string sequences which would be problematic for eval.
+      # TODO remove all characters not valid in ruby symbols
+      def sanitize_parameters(parameter_string)
+        parameter_string.gsub(/#</, '"').gsub(/>,/, '", ').gsub(/\\0/, '')
+      end
+
       # Slow default method to parse timestamps.
       # Reimplement this function in a file format specific Request class
       # to improve the timestamp parsing speed.
@@ -177,4 +191,4 @@ module RequestLogAnalyzer
       @lines.map { |line| line[:lineno] }.reject { |v| v.nil? }.max
     end
   end
-end
+end

data/lib/request_log_analyzer/source/log_parser.rb

@@ -43,7 +43,9 @@ module RequestLogAnalyzer::Source
       @progress_handler = nil
 
       @options[:parse_strategy] ||= DEFAULT_PARSE_STRATEGY
-      raise "Unknown parse strategy" unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
+      unless PARSE_STRATEGIES.include?(@options[:parse_strategy])
+        raise "Unknown parse strategy: #{@options[@parse_strategy]}"
+      end
     end
 
     # Reads the input, which can either be a file, sequence of files or STDIN to parse

data/lib/request_log_analyzer/source.rb

@@ -8,12 +8,8 @@
 # - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
 module RequestLogAnalyzer::Source
 
-  # Loads constants that reside in the RequestLogAnalyzer::Source namespace. This function uses
-  # RequestLogAnalyzer::load_default_class_file to load the file in which the constant is declared.
-  # <tt>const</tt>:: The constant to load in the RequestLogAnalyzer::Source namespace.
-  def self.const_missing(const)
-    RequestLogAnalyzer::load_default_class_file(self, const)
-  end
+  autoload :LogParser,      'request_log_analyzer/source/log_parser'
+  autoload :DatabaseLoader, 'request_log_analyzer/source/database_loader'
 
   # The base Source class. All other sources should inherit from this class.
   #

data/lib/request_log_analyzer/tracker/duration.rb

@@ -25,6 +25,13 @@ module RequestLogAnalyzer::Tracker
     def prepare
       options[:value] = options[:duration] if options[:duration]
       super
+      
+      @number_of_buckets = options[:number_of_buckets] || 1000
+      @min_bucket_value  = options[:min_bucket_value] ? options[:min_bucket_value].to_f : 0.0001
+      @max_bucket_value  = options[:max_bucket_value] ? options[:max_bucket_value].to_f : 1000
+
+      # precalculate the bucket size
+      @bucket_size = (Math.log(@max_bucket_value) - Math.log(@min_bucket_value)) / @number_of_buckets.to_f
     end
 
     # Display a duration