request-log-analyzer 1.10.1 → 1.11.0

data/bin/request-log-analyzer

@@ -58,7 +58,6 @@ begin
 
     command_line.switch(:debug)
     command_line.switch(:no_progress)
-    command_line.switch(:gets_memory_protection)
     command_line.switch(:silent)
 
     command_line.minimum_parameters = 1

data/lib/request_log_analyzer.rb

@@ -10,36 +10,10 @@ Encoding.default_external = 'binary' if defined? Encoding and Encoding.respond_t
 # The {RequestLogAnalyzer::VERSION} constant can be used to determine what version of request-log-analyzer
 # is running.
 module RequestLogAnalyzer
-
+  
   # The current version of request-log-analyzer.
   # Do not change the value by hand; it will be updated automatically by the gem release script.
-  VERSION = "1.10.1"
-
-
-  autoload :Controller,     'request_log_analyzer/controller'
-  autoload :Aggregator,     'request_log_analyzer/aggregator'
-  autoload :Database,       'request_log_analyzer/database'
-  autoload :FileFormat,     'request_log_analyzer/file_format'
-  autoload :Filter,         'request_log_analyzer/filter'
-  autoload :LineDefinition, 'request_log_analyzer/line_definition'
-  autoload :LogProcessor,   'request_log_analyzer/log_processor'
-  autoload :Mailer,         'request_log_analyzer/mailer'
-  autoload :Output,         'request_log_analyzer/output'
-  autoload :Request,        'request_log_analyzer/request'
-  autoload :Source,         'request_log_analyzer/source'
-  autoload :Tracker,        'request_log_analyzer/tracker'
-
-  # Loads constants that reside in the RequestLogAnalyzer tree using the constant name and its base 
-  # constant to determine the filename.
-  # @param [Module] base The base constant to load the constant from. This should be <tt>Foo</tt> when
-  #        the constant <tt>Foo::Bar</tt> is being loaded.
-  # @param [Symbol] const The constant to load from the base constant as a string or symbol. This 
-  #        should be <tt>"Bar"<tt> or <tt>:Bar</tt> when the constant <tt>Foo::Bar</tt> is being loaded.
-  # @return [Module] The loaded module, nil if it was not found on the expected location.
-  def self.load_default_class_file(base, const)
-    require "#{to_underscore("#{base.name}::#{const}")}"
-    base.const_get(const) if base.const_defined?(const)
-  end
+  VERSION = "1.11.0"
 
   # Convert a string/symbol in camelcase ({RequestLogAnalyzer::Controller}) to underscores 
   # (<tt>request_log_analyzer/controller</tt>). This function can be used to load the file (using 
@@ -55,9 +29,21 @@ module RequestLogAnalyzer
   # ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given 
   # filename.
   #
-  # @param [#to_s] str The string-like to convert in the following format: <tt>module_name/class_name</tt>.
+  # @param [#to_s] str The string-like to convert in the f`ollowing format: <tt>module_name/class_name</tt>.
   # @return [String] The input string converted to camelcase form.  
   def self.to_camelcase(str)
     str.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
   end
 end
+
+require 'request_log_analyzer/controller'
+require 'request_log_analyzer/aggregator'
+require 'request_log_analyzer/file_format'
+require 'request_log_analyzer/filter'
+require 'request_log_analyzer/line_definition'
+require 'request_log_analyzer/log_processor'
+require 'request_log_analyzer/mailer'
+require 'request_log_analyzer/output'
+require 'request_log_analyzer/request'
+require 'request_log_analyzer/source'
+require 'request_log_analyzer/tracker'

data/lib/request_log_analyzer/aggregator.rb

@@ -1,9 +1,5 @@
 module RequestLogAnalyzer::Aggregator
 
-  autoload :Echo,             'request_log_analyzer/aggregator/echo'
-  autoload :Summarizer,       'request_log_analyzer/aggregator/summarizer'
-  autoload :DatabaseInserter, 'request_log_analyzer/aggregator/database_inserter'
-
   # The base class of an aggregator. This class provides the interface to which
   # every aggregator should comply (by simply subclassing this class).
   class Base
@@ -46,4 +42,8 @@ module RequestLogAnalyzer::Aggregator
     end
 
   end
-end
+end
+
+require 'request_log_analyzer/aggregator/echo'
+require 'request_log_analyzer/aggregator/summarizer'
+require 'request_log_analyzer/aggregator/database_inserter'

data/lib/request_log_analyzer/aggregator/database_inserter.rb

@@ -1,4 +1,3 @@
-
 module RequestLogAnalyzer::Aggregator
 
   # The database aggregator will create an SQLite3 database with all parsed request information.
@@ -19,6 +18,8 @@ module RequestLogAnalyzer::Aggregator
     # Establishes a connection to the database and creates the necessary database schema for the
     # current file format
     def prepare
+      require 'request_log_analyzer/database'
+      
       @sources = {}
       @database = RequestLogAnalyzer::Database.new(options[:database])
       @database.file_format = source.file_format

data/lib/request_log_analyzer/controller.rb

@@ -21,8 +21,6 @@ module RequestLogAnalyzer
     # <tt>arguments<tt> A CommandLine::Arguments hash containing parsed commandline parameters.
     def self.build_from_arguments(arguments)
 
-      require 'mixins/gets_memory_protection' if arguments[:gets_memory_protection]
-
       options = {}
 
       # Copy fields
@@ -58,7 +56,6 @@ module RequestLogAnalyzer
 
       # Handle output format casing
       if options[:output].class == String
-        options[:output] = 'FancyHTML'  if options[:output] =~ /^fancy_?html$/i
         options[:output] = 'HTML'       if options[:output] =~ /^html$/i
         options[:output] = 'FixedWidth' if options[:output] =~ /^fixed_?width$/i
       end

data/lib/request_log_analyzer/database.rb

@@ -3,12 +3,7 @@ require 'active_record'
 
 class RequestLogAnalyzer::Database
 
-  autoload :Connection, 'request_log_analyzer/database/connection'
-  autoload :Base,       'request_log_analyzer/database/base'
-  autoload :Request,    'request_log_analyzer/database/request'
-  autoload :Source,     'request_log_analyzer/database/source'
-  autoload :Warning,    'request_log_analyzer/database/warning'
-
+  require 'request_log_analyzer/database/connection'
   include RequestLogAnalyzer::Database::Connection
 
   attr_accessor :file_format
@@ -100,5 +95,9 @@ class RequestLogAnalyzer::Database
       end
     end
   end
-
 end
+
+require 'request_log_analyzer/database/base'
+require 'request_log_analyzer/database/request'
+require 'request_log_analyzer/database/source'
+require 'request_log_analyzer/database/warning'

data/lib/request_log_analyzer/file_format.rb

@@ -121,6 +121,18 @@ module RequestLogAnalyzer::FileFormat
       'Z' => '(?:[+-]\d{4}|[A-Z]{3,4})',
       '%' => '%'
     }
+    
+    # Creates a regular expression to match a hostname
+    def hostname(blank = false)
+      regexp = /(?:(?:[a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*(?:[A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])/
+      add_blank_option(regexp, blank)
+    end
+    
+    # Creates a regular expression to match a hostname or ip address
+    def hostname_or_ip_address(blank = false)
+      regexp = Regexp.union(hostname, ip_address)
+      add_blank_option(regexp, blank)
+    end
 
     # Create a regular expression for a timestamp, generated by a strftime call.
     # Provide the format string to construct a matching regular expression.
@@ -139,12 +151,7 @@ module RequestLogAnalyzer::FileFormat
         end
       end
 
-      regexp = Regexp.new(regexp)
-      return case blank
-        when String then Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
-        when true   then Regexp.union(regexp, //)
-        else regexp
-      end
+      add_blank_option(Regexp.new(regexp), blank)
     end
 
     # Construct a regular expression to parse IPv4 and IPv6 addresses.
@@ -162,12 +169,22 @@ module RequestLogAnalyzer::FileFormat
       ipv6_regex_compressed_hex_4_dec = /(?:(?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?)::(?:(?:[0-9A-Fa-f]{1,4}:)*)#{ipv4_regexp}/
       ipv6_regexp                     = Regexp.union(ipv6_regex_8_hex, ipv6_regex_compressed_hex, ipv6_regex_6_hex_4_dec, ipv6_regex_compressed_hex_4_dec)
 
-      # Allow the field to be blank if this option is given. This can be true to 
-      # allow an empty string or a string alternative for the nil value.
-      ip_regexp = case blank
-        when String then Regexp.union(ipv4_regexp, ipv6_regexp, Regexp.new(Regexp.quote(blank)))
-        when true   then Regexp.union(ipv4_regexp, ipv6_regexp, //)
-        else             Regexp.union(ipv4_regexp, ipv6_regexp)
+      add_blank_option(Regexp.union(ipv4_regexp, ipv6_regexp), blank)
+    end
+    
+    def anchored(regexp)
+      /^#{regexp}$/
+    end
+    
+    protected
+    
+    # Allow the field to be blank if this option is given. This can be true to 
+    # allow an empty string or a string alternative for the nil value.
+    def add_blank_option(regexp, blank)
+      case blank
+        when String; Regexp.union(regexp, Regexp.new(Regexp.quote(blank)))
+        when true;   Regexp.union(regexp, //)
+        else regexp
       end
     end
   end
@@ -260,9 +277,12 @@ module RequestLogAnalyzer::FileFormat
     end
 
     # Checks whether the file format is valid so it can be safely used with RLA.
-    def valid?
+    def well_formed?
       valid_line_definitions? && valid_request_class?
     end
+    
+    alias_method :valid?, :well_formed?
+  
 
     # Checks whether the line definitions form a valid language.
     # A file format should have at least a header and a footer line type    
@@ -294,5 +314,14 @@ module RequestLogAnalyzer::FileFormat
 
       return nil
     end
+    
+    # Returns the max line length for this file format if any.
+    def max_line_length
+      self.class.const_get(MAX_LINE_LENGTH) if self.class.const_defined?(:MAX_LINE_LENGTH)
+    end
+    
+    def line_divider
+      self.class.const_get(LINE_DIVIDER) if self.class.const_defined?(:LINE_DIVIDER)
+    end
   end
 end

data/lib/request_log_analyzer/file_format/apache.rb

@@ -31,7 +31,7 @@ module RequestLogAnalyzer::FileFormat
     # A hash that defines how the log format directives should be parsed.
     LOG_DIRECTIVES = {
       '%' => { nil => { :regexp => '%', :captures => [] } },
-      'h' => { nil => { :regexp => '([A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+)',  :captures => [{:name => :remote_host, :type => :string}] } },
+      'h' => { nil => { :regexp => "(#{hostname_or_ip_address})",  :captures => [{:name => :remote_host, :type => :string}] } },
       'a' => { nil => { :regexp => "(#{ip_address})", :captures => [{:name => :remote_ip, :type => :string}] } },
       'b' => { nil => { :regexp => '(\d+|-)', :captures => [{:name => :bytes_sent, :type => :traffic}] } },
       'c' => { nil => { :regexp => '(\+|\-|\X)', :captures => [{:name => :connection_status, :type => :integer}] } },

data/lib/request_log_analyzer/file_format/delayed_job2.rb

@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_lock do |line|
       line.header = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] acquired lock on (\S+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
       
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_completed do |line|
       line.footer = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(\S+) pid:(\d+) completed after (\d+\.\d+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
       line.capture(:pid).as(:integer)

data/lib/request_log_analyzer/file_format/delayed_job21.rb

@@ -8,7 +8,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_lock do |line|
       line.header = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] acquired lock on (\S+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/
       
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
@@ -18,7 +18,7 @@ module RequestLogAnalyzer::FileFormat
     
     line_definition :job_completed do |line|
       line.footer = true
-      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(\S+) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
+      line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/
       line.capture(:timestamp).as(:timestamp)
       line.capture(:host)
       line.capture(:pid).as(:integer)

data/lib/request_log_analyzer/file_format/haproxy.rb

@@ -4,31 +4,45 @@ module RequestLogAnalyzer::FileFormat
 
     extend CommonRegularExpressions
 
-    # Define line types
-    line_definition :haproxy do |line|
-      line.header = true
-      line.footer = true
-
-      line.regexp = %r{
+    # substitute version specific parts of the haproxy entry regexp.
+    def self.compose_regexp(millisecs, backends, counters, connections, queues)
+      %r{
         (#{ip_address}):\d+\s # client_ip ':' client_port
-        \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})\.\d{3}\]\s # '[' accept_date ']'
+        \[(#{timestamp('%d/%b/%Y:%H:%M:%S')})#{millisecs}\]\s # '[' accept_date ']'
         (\S+)\s # frontend_name
-        (\S+)\/(\S+)\s # backend_name '/' server_name
-        (\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        #{backends}
+        #{counters}
         (\d+)\s # status_code
         \+?(\d+)\s # bytes_read
         (\S+)\s # captured_request_cookie
         (\S+)\s # captured_response_cookie
         (\w|-)(\w|-)(\w|-)(\w|-)\s # termination_state
-        (\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
-        (\d+)\/(\d+)\s # srv_queue '/' backend_queue
+        #{connections}
+        #{queues}
         (\S*)\s? # captured_request_headers
         (\S*)\s? # captured_response_headers
         "([^"]*)" # '"' http_request '"'
       }x
+    end
+
+    # Define line types
+
+    # line definition for haproxy 1.3 and higher
+    line_definition :haproxy13 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /\.\d{3}\] \S+ \S+\/\S+ / # .millisecs] frontend_name backend_name/server_name
+
+      line.regexp = compose_regexp(
+        '\.\d{3}', # millisecs
+        '(\S+)\/(\S+)\s', # backend_name '/' server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\/(\d+)\/(\d+)\/\+?(\d+)\s', # actconn '/' feconn '/' beconn '/' srv_conn '/' retries
+        '(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
+      )
 
       line.capture(:client_ip).as(:string)
-      line.capture(:accept_date).as(:timestamp)
+      line.capture(:timestamp).as(:timestamp)
       line.capture(:frontend_name).as(:string)
       line.capture(:backend_name).as(:string)
       line.capture(:server_name).as(:string)
@@ -57,9 +71,89 @@ module RequestLogAnalyzer::FileFormat
       line.capture(:http_request).as(:nillable_string)
     end
 
+    # haproxy 1.2 has a few fields less than 1.3+
+    line_definition :haproxy12 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /\.\d{3}\] \S+ \S+ / # .millisecs] frontend_name server_name
+
+      line.regexp = compose_regexp(
+        '\.\d{3}', # millisecs
+        '(\S+)\s', # server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tw '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\/(\d+)\s', # srv_conn '/' listener_conn '/' process_conn
+        '(\d+)\/(\d+)\s' # srv_queue '/' backend_queue
+      )
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tw).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:srv_conn).as(:integer)
+      line.capture(:listener_conn).as(:integer)
+      line.capture(:process_conn).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
+    # and haproxy 1.1 has even less fields
+    line_definition :haproxy11 do |line|
+      line.header = true
+      line.footer = true
+      line.teaser = /:\d{2}\] \S+ \S+ / # :secs] frontend_name server_name
+
+      line.regexp = compose_regexp(
+        '', # no millisec precision in this version of haproxy
+        '(\S+)\s', # server_name
+        '(\d+|-1)\/(\d+|-1)\/(\d+|-1)\/\+?(\d+)\s', # Tq '/' Tc '/' Tr '/' Tt
+        '(\d+)\/(\d+)\s', # listener_conn '/' process_conn
+        '' # no queues in this version of haproxy
+      )
+
+      line.capture(:client_ip).as(:string)
+      line.capture(:timestamp).as(:timestamp)
+      line.capture(:frontend_name).as(:string)
+      line.capture(:server_name).as(:string)
+      line.capture(:tq).as(:nillable_duration, :unit => :msec)
+      line.capture(:tc).as(:nillable_duration, :unit => :msec)
+      line.capture(:tr).as(:nillable_duration, :unit => :msec)
+      line.capture(:tt).as(:duration, :unit => :msec)
+      line.capture(:status_code).as(:integer)
+      line.capture(:bytes_read).as(:traffic, :unit => :byte)
+      line.capture(:captured_request_cookie).as(:nillable_string)
+      line.capture(:captured_response_cookie).as(:nillable_string)
+      line.capture(:termination_event_code).as(:nillable_string)
+      line.capture(:terminated_session_state).as(:nillable_string)
+      line.capture(:clientside_persistence_cookie).as(:nillable_string)
+      line.capture(:serverside_persistence_cookie).as(:nillable_string)
+      line.capture(:listener_conn).as(:integer)
+      line.capture(:process_conn).as(:integer)
+      line.capture(:srv_queue).as(:integer)
+      line.capture(:backend_queue).as(:integer)
+      line.capture(:captured_request_headers).as(:nillable_string)
+      line.capture(:captured_response_headers).as(:nillable_string)
+      line.capture(:http_request).as(:nillable_string)
+    end
+
     # Define the summary report
     report do |analyze|
-      analyze.hourly_spread :field => :accept_date
+      analyze.hourly_spread :field => :timestamp
 
       analyze.frequency :client_ip,
         :title => "Hits per IP"

data/lib/request_log_analyzer/file_format/mysql.rb

@@ -15,7 +15,7 @@ module RequestLogAnalyzer::FileFormat
     line_definition :user_host do |line|
       line.header = :alternative
       line.teaser = /\# User\@Host\: /
-      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ ([\w\.-]*) \[(#{ip_address(true)})\]/
+      line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ (#{hostname(true)}) \[(#{ip_address(true)})\]/
       
       line.capture(:user)
       line.capture(:host)
@@ -34,18 +34,18 @@ module RequestLogAnalyzer::FileFormat
     end
 
     line_definition :use_database do |line|
-      line.regexp   = /^use (\w+);\s*$/
+      line.regexp   = /^\s*use (\w+);\s*$/
       line.capture(:database)
     end
 
     line_definition :query_part do |line|
-      line.regexp   = /^(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
+      line.regexp   = /^\s*(?!(?:use |\# |SET ))(.*[^;\s])\s*$/
       line.capture(:query_fragment)
     end
 
     line_definition :query do |line|
       line.footer = true
-      line.regexp = /^(?!(?:use |\# |SET ))(.*);\s*$/
+      line.regexp = /^(?!\s*(?:use |\# |SET ))(.*);\s*$/
       line.capture(:query).as(:sql)
     end
 
@@ -84,7 +84,7 @@ module RequestLogAnalyzer::FileFormat
         sql.gsub!(/(:int,)+:int/, ':ints')                                # replace multiple ints by a list
         sql.gsub!(/(:string,)+:string/, ':strings')                       # replace multiple strings by a list
 
-        return sql.rstrip
+        return sql.strip
       end
 
       def host