request-log-analyzer 1.10.0 → 1.11.0

data/.gitignore

@@ -10,4 +10,5 @@ requests.db
 /files
 /coverage
 Gemfile.lock
-/.bundle/
+/.bundle/
+*.rbc

data/.infinity_test

@@ -0,0 +1,8 @@
+infinity_test do
+  
+  use :rubies => %w(1.8.7 1.9.2 ree), :test_framework => :rspec   # jruby rbx
+  
+  before(:each_ruby) do |environment|
+    environment.system('bundle install')
+  end
+end

data/bin/request-log-analyzer

@@ -58,7 +58,6 @@ begin
 
     command_line.switch(:debug)
     command_line.switch(:no_progress)
-    command_line.switch(:gets_memory_protection)
     command_line.switch(:silent)
 
     command_line.minimum_parameters = 1

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -1,4 +1,3 @@
-
 module RequestLogAnalyzer::Aggregator
 
   # The database aggregator will create an SQLite3 database with all parsed request information.
@@ -19,6 +18,8 @@ module RequestLogAnalyzer::Aggregator
     # Establishes a connection to the database and creates the necessary database schema for the
     # current file format
     def prepare
+      require 'request_log_analyzer/database'
+      
       @sources = {}
       @database = RequestLogAnalyzer::Database.new(options[:database])
       @database.file_format = source.file_format

data/lib/request_log_analyzer/aggregator.rb

@@ -1,9 +1,5 @@
 module RequestLogAnalyzer::Aggregator
 
-  autoload :Echo,             'request_log_analyzer/aggregator/echo'
-  autoload :Summarizer,       'request_log_analyzer/aggregator/summarizer'
-  autoload :DatabaseInserter, 'request_log_analyzer/aggregator/database_inserter'
-
   # The base class of an aggregator. This class provides the interface to which
   # every aggregator should comply (by simply subclassing this class).
   class Base
@@ -46,4 +42,8 @@ module RequestLogAnalyzer::Aggregator
     end
 
   end
-end
+end
+
+require 'request_log_analyzer/aggregator/echo'
+require 'request_log_analyzer/aggregator/summarizer'
+require 'request_log_analyzer/aggregator/database_inserter'

data/lib/request_log_analyzer/controller.rb

@@ -21,8 +21,6 @@ module RequestLogAnalyzer
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
     def self.build_from_arguments(arguments)
 
-      require 'mixins/gets_memory_protection' if arguments[:gets_memory_protection]
-
       options = {}
 
       # Copy fields
@@ -58,7 +56,6 @@ module RequestLogAnalyzer
 
       # Handle output format casing
       if options[:output].class == String
-        options[:output] = 'FancyHTML'  if options[:output] =~ /^fancy_?html$/i
         options[:output] = 'HTML'       if options[:output] =~ /^html$/i
         options[:output] = 'FixedWidth' if options[:output] =~ /^fixed_?width$/i
       end

data/lib/request_log_analyzer/database.rb

@@ -3,12 +3,7 @@ require 'active_record'
 
 class RequestLogAnalyzer::Database
 
-  autoload :Connection, 'request_log_analyzer/database/connection'
-  autoload :Base,       'request_log_analyzer/database/base'
-  autoload :Request,    'request_log_analyzer/database/request'
-  autoload :Source,     'request_log_analyzer/database/source'
-  autoload :Warning,    'request_log_analyzer/database/warning'
-
+  require 'request_log_analyzer/database/connection'
   include RequestLogAnalyzer::Database::Connection
 
   attr_accessor :file_format
@@ -100,5 +95,9 @@ class RequestLogAnalyzer::Database
       end
     end
   end
-
 end
+
+require 'request_log_analyzer/database/base'
+require 'request_log_analyzer/database/request'
+require 'request_log_analyzer/database/source'
+require 'request_log_analyzer/database/warning'

data/lib/request_log_analyzer/file_format/apache.rb

@@ -31,7 +31,7 @@ module RequestLogAnalyzer::FileFormat
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
       '%' => { nil => { :regexp => '%', :captures => [] } },
-      'h' => { nil => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] } },
+      'h' => { nil => { :regexp => "(#{hostname_or_ip_address})",  :captures => [{:name => :remote_host, :type => :string}] } },
       'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
       'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
       'c' => { nil => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] } },

data/lib/request_log_analyzer/file_format/delayed_job2.rb

@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_lock do |line|
       line.header = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] acquired lock on (\S+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
       
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_completed do |line|
       line.footer = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(\S+) pid:(\d+) completed after (\d+\.\d+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
       line.capture(:pid).as(:integer)

data/lib/request_log_analyzer/file_format/delayed_job21.rb

@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_lock do |line|
       line.header = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] acquired lock on (\S+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
       
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_completed do |line|
       line.footer = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
       line.capture(:pid).as(:integer)

data/lib/request_log_analyzer/file_format/haproxy.rb

@@ -4,31 +4,45 @@ module RequestLogAnalyzer::FileFormat
 
     extend CommonRegularExpressions
 
-    # Define line types
-    line_definition :haproxy do |line|
-      line.header = true
-      line.footer = true
-
-      line.regexp = %r{
+    # substitute version specific parts of the haproxy entry regexp.
+    def self.compose_regexp(millisecs, backends, counters, connections, queues)
+      %r{
         (#{ip_address}):\d+\s # client_ip ':' client_port
-        \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})\.\d{3}\]\s # '[' accept_date ']'
+        \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})#{millisecs}\]\s # '[' accept_date ']'
         (\S+)\s # frontend_name
-        (\S+)\/(\S+)\s # backend_name '/' server_name
-        (\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        #{backends}
+        #{counters}
         (\d+)\s # status_code
         \+?(\d+)\s # bytes_read
         (\S+)\s # captured_request_cookie
         (\S+)\s # captured_response_cookie
         (\w|-)(\w|-)(\w|-)(\w|-)\s # termination_state
-        (\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
-        (\d+)\/(\d+)\s # srv_queue '/' backend_queue
+        #{connections}
+        #{queues}
         (\S*)\s? # captured_request_headers
         (\S*)\s? # captured_response_headers
         "([^"]*)" # '"' http_request '"'
       }x
+    end
+
+    # Define line types
+
+    # line definition for haproxy 1.3 and higher
+    line_definition :haproxy13 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /\.\d{3}\] \S+ \S+\/\S+ / # .millisecs] frontend_name backend_name/server_name
+
+      line.regexp = compose_regexp(
+        '\.\d{3}', # millisecs
+        '(\S+)\/(\S+)\s', # backend_name '/' server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s', # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
+        '(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
+      )
 
       line.capture(:client_ip).as(:string)
-      line.capture(:accept_date).as(:timestamp)
+      line.capture(:timestamp).as(:timestamp)
       line.capture(:frontend_name).as(:string)
       line.capture(:backend_name).as(:string)
       line.capture(:server_name).as(:string)
@@ -57,9 +71,89 @@ module RequestLogAnalyzer::FileFormat
       line.capture(:http_request).as(:nillable_string)
     end
 
+    # haproxy 1.2 has a few fields less than 1.3+
+    line_definition :haproxy12 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /\.\d{3}\] \S+ \S+ / # .millisecs] frontend_name server_name
+
+      line.regexp = compose_regexp(
+        '\.\d{3}', # millisecs
+        '(\S+)\s', # server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\/(\d+)\s', # srv_conn '/' listener_conn '/' process_conn
+        '(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
+      )
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tw).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:srv_conn).as(:integer)
+      line.capture(:listener_conn).as(:integer)
+      line.capture(:process_conn).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
+    # and haproxy 1.1 has even less fields
+    line_definition :haproxy11 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /:\d{2}\] \S+ \S+ / # :secs] frontend_name server_name
+
+      line.regexp = compose_regexp(
+        '', # no millisec precision in this version of haproxy
+        '(\S+)\s', # server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\s', # listener_conn '/' process_conn
+        '' # no queues in this version of haproxy
+      )
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:listener_conn).as(:integer)
+      line.capture(:process_conn).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
     # Define the summary report
     report do |analyze|
-      analyze.hourly_spread :field => :accept_date
+      analyze.hourly_spread :field => :timestamp
 
       analyze.frequency :client_ip,
         :title => "Hits per IP"

data/lib/request_log_analyzer/file_format/mysql.rb

@@ -15,7 +15,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :user_host do |line|
       line.header = :alternative
       line.teaser = /\# User\@Host\: /
-      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[(#{ip_address(true)})\]/
+      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ (#{hostname(true)}) \[(#{ip_address(true)})\]/
       
       line.capture(:user)
       line.capture(:host)
@@ -34,18 +34,18 @@ module RequestLogAnalyzer::FileFormat
     end
 
     line_definition :use_database do |line|
-      line.regexp   = /^use (\w+);\s*$/
+      line.regexp   = /^\s*use (\w+);\s*$/
       line.capture(:database)
     end
 
     line_definition :query_part do |line|
-      line.regexp   = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
+      line.regexp   = /^\s*(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
       line.capture(:query_fragment)
     end
 
     line_definition :query do |line|
       line.footer = true
-      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
+      line.regexp = /^(?!\s*(?:use |\# |SET ))(.*);\s*$/
       line.capture(:query).as(:sql)
     end
 
@@ -84,7 +84,7 @@ module RequestLogAnalyzer::FileFormat
         sql.gsub!(/(:int,)+:int/, ':ints')                                # replace multiple ints by a list
         sql.gsub!(/(:string,)+:string/, ':strings')                       # replace multiple strings by a list
 
-        return sql.rstrip
+        return sql.strip
       end
 
       def host

data/lib/request_log_analyzer/file_format/rails3.rb

@@ -23,12 +23,19 @@ module RequestLogAnalyzer::FileFormat
     # Processing by QueriesController#index as HTML
     line_definition :processing do |line|
       line.teaser = /Processing by /
-      line.regexp = /Processing by ([A-Za-z0-9\-:]+)\#(\w+) as ([\w\/\*]+)/
+      line.regexp = /Processing by ([A-Za-z0-9\-:]+)\#(\w+) as ([\w\/\*]*)/
       
       line.capture(:controller)
       line.capture(:action)
       line.capture(:format)
     end
+
+    # Parameters: {"action"=>"cached", "controller"=>"cached"}
+    line_definition :parameters do |line|
+      line.teaser   = /  Parameters:/
+      line.regexp   = /  Parameters:\s+(\{.*\})/
+      line.capture(:params).as(:eval)
+    end
     
     # Completed 200 OK in 224ms (Views: 200.2ms | ActiveRecord: 3.4ms)
     # Completed 302 Found in 23ms

data/lib/request_log_analyzer/file_format.rb

@@ -34,7 +34,7 @@ module RequestLogAnalyzer::FileFormat
 
     elsif file_format.kind_of?(String) && File.exist?(file_format) && File.file?(file_format)
       # load a format from a ruby file
-      require file_format
+      require File.expand_path(file_format)
 
       const = RequestLogAnalyzer.to_camelcase(File.basename(file_format, '.rb'))
       if RequestLogAnalyzer::FileFormat.const_defined?(const)
@@ -121,6 +121,18 @@ module RequestLogAnalyzer::FileFormat
       'Z' => '(?:[+-]\d{4}|[A-Z]{3,4})',
       '%' => '%'
     }
+    
+    # Creates a regular expression to match a hostname
+    def hostname(blank = false)
+      regexp = /(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])/
+      add_blank_option(regexp, blank)
+    end
+    
+    # Creates a regular expression to match a hostname or ip address
+    def hostname_or_ip_address(blank = false)
+      regexp = Regexp.union(hostname, ip_address)
+      add_blank_option(regexp, blank)
+    end
 
     # Create a regular expression for a timestamp, generated by a strftime call.
     # Provide the format string to construct a matching regular expression.
@@ -139,12 +151,7 @@ module RequestLogAnalyzer::FileFormat
         end
       end
 
-      regexp = Regexp.new(regexp)
-      return case blank
-        when String then Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
-        when true   then Regexp.union(regexp, //)
-        else regexp
-      end
+      add_blank_option(Regexp.new(regexp), blank)
     end
 
     # Construct a regular expression to parse IPv4 and IPv6 addresses.
@@ -162,12 +169,22 @@ module RequestLogAnalyzer::FileFormat
       ipv6_regex_compressed_hex_4_dec = /(?:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::(?:(?:[0-9A-Fa-f]{1,4}:)*)#{ipv4_regexp}/
       ipv6_regexp                     = Regexp.union(ipv6_regex_8_hex, ipv6_regex_compressed_hex, ipv6_regex_6_hex_4_dec, ipv6_regex_compressed_hex_4_dec)
 
-      # Allow the field to be blank if this option is given. This can be true to 
-      # allow an empty string or a string alternative for the nil value.
-      ip_regexp = case blank
-        when String then Regexp.union(ipv4_regexp, ipv6_regexp, Regexp.new(Regexp.quote(blank)))
-        when true   then Regexp.union(ipv4_regexp, ipv6_regexp, //)
-        else             Regexp.union(ipv4_regexp, ipv6_regexp)
+      add_blank_option(Regexp.union(ipv4_regexp, ipv6_regexp), blank)
+    end
+    
+    def anchored(regexp)
+      /^#{regexp}$/
+    end
+    
+    protected
+    
+    # Allow the field to be blank if this option is given. This can be true to 
+    # allow an empty string or a string alternative for the nil value.
+    def add_blank_option(regexp, blank)
+      case blank
+        when String; Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
+        when true;   Regexp.union(regexp, //)
+        else regexp
       end
     end
   end
@@ -260,9 +277,12 @@ module RequestLogAnalyzer::FileFormat
     end
 
     # Checks whether the file format is valid so it can be safely used with RLA.
-    def valid?
+    def well_formed?
       valid_line_definitions? && valid_request_class?
     end
+    
+    alias_method :valid?, :well_formed?
+  
 
     # Checks whether the line definitions form a valid language.
     # A file format should have at least a header and a footer line type    
@@ -294,5 +314,14 @@ module RequestLogAnalyzer::FileFormat
 
       return nil
     end
+    
+    # Returns the max line length for this file format if any.
+    def max_line_length
+      self.class.const_get(MAX_LINE_LENGTH) if self.class.const_defined?(:MAX_LINE_LENGTH)
+    end
+    
+    def line_divider
+      self.class.const_get(LINE_DIVIDER) if self.class.const_defined?(:LINE_DIVIDER)
+    end
   end
 end

data/lib/request_log_analyzer/filter.rb

@@ -1,9 +1,5 @@
 module RequestLogAnalyzer::Filter
 
-  autoload :Field,     'request_log_analyzer/filter/field'
-  autoload :Timespan,  'request_log_analyzer/filter/timespan'
-  autoload :Anonymize, 'request_log_analyzer/filter/anonymize'
-
   # Base filter class used to filter input requests.
   # All filters should interit from this base.
   class Base
@@ -24,5 +20,8 @@ module RequestLogAnalyzer::Filter
       request
     end
   end
+end
 
-end
+require 'request_log_analyzer/filter/field'
+require 'request_log_analyzer/filter/timespan'
+require 'request_log_analyzer/filter/anonymize'

data/lib/request_log_analyzer/line_definition.rb

@@ -70,6 +70,10 @@ module RequestLogAnalyzer
       captures << new_capture_hash
       CaptureDefiner.new(new_capture_hash)
     end
+    
+    def all_captured_variables
+      captures.map { |c| c[:name] } + captures.map { |c| c[:provides] }.compact.map { |pr| pr.keys }.flatten
+    end
 
     # Checks whether a given line matches this definition.
     # It will return false if a line does not match. If the line matches, a hash is returned
@@ -128,9 +132,7 @@ module RequestLogAnalyzer
 
     # Returns true if this line captures values of the given name
     def captures?(name)
-      captures.any? { |c| c[:name] == name }
+      all_captured_variables.include?(name)
     end
-
   end
-
-end
+end

data/lib/request_log_analyzer/output.rb

@@ -1,10 +1,6 @@
 # Module for generating output
 module RequestLogAnalyzer::Output
 
-  autoload :FixedWidth, 'request_log_analyzer/output/fixed_width'
-  autoload :HTML,       'request_log_analyzer/output/html'
-  autoload :FancyHTML,  'request_log_analyzer/output/fancy_html'
-  
   # Loads a Output::Base subclass instance.
   def self.load(file_format, *args)
     
@@ -111,6 +107,8 @@ module RequestLogAnalyzer::Output
     def table_has_header?(columns)
       columns.any? { |column| !column[:title].nil? }
     end
-
   end
 end
+
+require 'request_log_analyzer/output/fixed_width'
+require 'request_log_analyzer/output/html'

data/lib/request_log_analyzer/source/log_parser.rb

@@ -13,6 +13,11 @@ module RequestLogAnalyzer::Source
 
     include Enumerable
 
+    # The maximum number of bytes to read from a line.
+    DEFAULT_MAX_LINE_LENGTH = 8096
+    
+    DEFAULT_LINE_DIVIDER = "\n"
+
     # The default parse strategy that will be used to parse the input.
     DEFAULT_PARSE_STRATEGY = 'assume-correct'
 
@@ -48,6 +53,14 @@ module RequestLogAnalyzer::Source
       end
     end
 
+    def max_line_length
+      file_format.max_line_length || DEFAULT_MAX_LINE_LENGTH
+    end
+    
+    def line_divider
+      file_format.line_divider || DEFAULT_LINE_DIVIDER
+    end
+
     # Reads the input, which can either be a file, sequence of files or STDIN to parse
     # lines specified in the FileFormat. This lines will be combined into Request instances,
     # that will be yielded. The actual parsing occurs in the parse_io method.
@@ -132,6 +145,13 @@ module RequestLogAnalyzer::Source
     def parse_stream(stream, options = {}, &block)
       parse_io(stream, options, &block)
     end
+    
+    # Parses a string. It will simply call parse_io. This function does not support progress updates.
+    # <tt>string</tt>:: The string that should be parsed.
+    # <tt>options</tt>:: A Hash of options that will be pased to parse_io.
+    def parse_string(string, options = {}, &block)
+      parse_io(StringIO.new(string), options, &block)
+    end
 
     # This method loops over each line of the input stream. It will try to parse this line as any of
     # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
@@ -143,11 +163,15 @@ module RequestLogAnalyzer::Source
     # - The method will yield progress updates if a progress handler is installed using progress=
     # - The method will yield parse warnings if a warning handler is installed using warning=
     #
+    # This is a Ruby 1.9 specific version that offers memory protection.
+    #
     # <tt>io</tt>:: The IO instance to use as source
     # <tt>options</tt>:: A hash of options that can be used by the parser.
-    def parse_io(io, options = {}, &block) # :yields: request
-      @current_lineno = 0
-      while line = io.gets
+    def parse_io_19(io, options = {}, &block) # :yields: request
+      @max_line_length = options[:max_line_length] || max_line_length
+      @line_divider    = options[:line_divider]    || line_divider
+      @current_lineno  = 0
+      while line = io.gets(@line_divider, @max_line_length)
         @current_lineno += 1
         @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
         parse_line(line, &block)
@@ -157,6 +181,35 @@ module RequestLogAnalyzer::Source
       @current_lineno = nil
     end
     
+    # This method loops over each line of the input stream. It will try to parse this line as any of
+    # the lines that are defined by the current file format (see RequestLogAnalyazer::FileFormat).
+    # It will then combine these parsed line into requests using heuristics. These requests (see
+    # RequestLogAnalyzer::Request) will then be yielded for further processing in the pipeline.
+    #
+    # - RequestLogAnalyzer::LineDefinition#matches is called to test if a line matches a line definition of the file format.
+    # - update_current_request is used to combine parsed lines into requests using heuristics.
+    # - The method will yield progress updates if a progress handler is installed using progress=
+    # - The method will yield parse warnings if a warning handler is installed using warning=
+    #
+    # This is a Ruby 1.8 specific version that doesn't offer memory protection.
+    #
+    # <tt>io</tt>:: The IO instance to use as source
+    # <tt>options</tt>:: A hash of options that can be used by the parser.    
+    def parse_io_18(io, options = {}, &block) # :yields: request
+      @line_divider    = options[:line_divider]    || line_divider
+      @current_lineno  = 0
+      while line = io.gets(@line_divider)
+        @current_lineno += 1
+        @progress_handler.call(:progress, io.pos) if @progress_handler && @current_lineno % 255 == 0
+        parse_line(line, &block)
+      end
+
+      warn(:unfinished_request_on_eof, "End of file reached, but last request was not completed!") unless @current_request.nil?
+      @current_lineno = nil
+    end
+    
+    alias_method :parse_io, RUBY_VERSION.to_f < 1.9 ? :parse_io_18 : :parse_io_19
+    
     # Parses a single line using the current file format. If successful, use the parsed
     # information to build a request
     # <tt>line</tt>:: The line to parse
@@ -295,5 +348,4 @@ module RequestLogAnalyzer::Source
       hash[:line_definition].footer
     end
   end
-
 end

data/lib/request_log_analyzer/source.rb

@@ -8,9 +8,6 @@
 # - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source.
 module RequestLogAnalyzer::Source
 
-  autoload :LogParser,      'request_log_analyzer/source/log_parser'
-  autoload :DatabaseLoader, 'request_log_analyzer/source/database_loader'
-
   # The base Source class. All other sources should inherit from this class.
   #
   # A source implememtation should at least implement the each_request method, which should yield
@@ -65,4 +62,6 @@ module RequestLogAnalyzer::Source
     end
 
   end
-end
+end
+
+require 'request_log_analyzer/source/log_parser'