ntail 0.0.6 → 0.0.7

data/lib/ntail/http_version.rb

@@ -0,0 +1,22 @@
+module NginxTail
+  module HttpVersion
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        def self.to_http_version_s(http_version)
+          http_version || "" # will be nil if $request == "-" (ie. "dodgy" HTTP requests)
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'http_version'" unless base.instance_methods.include? 'http_version'
+        
+      end
+    end
+
+    def to_http_version_s
+      self.class.to_http_version_s(self.http_version)
+    end
+    
+  end
+end

data/lib/ntail/known_ip_addresses.rb

@@ -0,0 +1,44 @@
+module NginxTail
+  module KnownIpAddresses
+
+    #
+    # known IP addresses, for filtering and formatting purposes
+    #
+    # e.g. office IP addresses, IP addresses of remote workers, ...
+    #
+
+    def self.included(base) # :nodoc:
+      base.class_eval do
+    
+        @@known_ip_addresses = []
+
+        # mainly (solely?) for testing purposes...
+        def self.known_ip_addresses()
+          @@known_ip_addresses.dup
+        end
+        
+        # mainly (solely?) for testing purposes...
+        def self.reset_known_ip_addresses()
+          while !@@known_ip_addresses.empty? ; @@known_ip_addresses.pop ; end
+        end
+        
+        def self.add_known_ip_address(known_ip_address)
+          (@@known_ip_addresses << known_ip_address).uniq!
+        end
+
+        def self.known_ip_address?(remote_addr)
+          @@known_ip_addresses.include?(remote_addr)
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'remote_addr'" unless base.instance_methods.include? 'remote_addr'
+        
+      end
+    end
+    
+    def known_ip_address?
+      self.class.known_ip_address?(self.remote_addr)
+    end
+    
+  end
+end

data/lib/ntail/local_ip_addresses.rb

@@ -0,0 +1,44 @@
+module NginxTail
+  module LocalIpAddresses
+    
+    #
+    # local IP addresses, for filtering and formatting purposes
+    #
+    # e.g. downstream proxy servers (nginx web servers -> passenger app servers)
+    #
+
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        @@local_ip_addresses = []
+        
+        # mainly (solely?) for testing purposes...
+        def self.local_ip_addresses()
+          @@local_ip_addresses.dup
+        end
+        
+        # mainly (solely?) for testing purposes...
+        def self.reset_local_ip_addresses()
+          while !@@local_ip_addresses.empty? ; @@local_ip_addresses.pop ; end
+        end
+        
+        def self.add_local_ip_address(local_ip_address)
+          (@@local_ip_addresses << local_ip_address).uniq!
+        end
+
+        def self.local_ip_address?(remote_addr)
+          @@local_ip_addresses.include?(remote_addr)
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'remote_addr'" unless base.instance_methods.include? 'remote_addr'
+        
+      end
+    end
+    
+    def local_ip_address?
+      self.class.local_ip_address?(self.remote_addr)
+    end
+    
+  end
+end

data/lib/ntail/log_line.rb

@@ -1,36 +1,54 @@
-require 'date'
-require 'socket'
 require 'net/http'
 
 require 'rubygems'
 require 'rainbow'
-require 'user-agent'
-
-begin
-  require 'geoip'
-rescue
-  # NOOP (optional dependency)
-end
 
 module NginxTail
   class LogLine
 
-    attr_accessor :raw_line
-    attr_accessor :parsable
+    def self.component_to_module_name(component)
+      # this mimicks the ActiveSupport::Inflector.camelize() method in Rails...
+      component.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    end
+    
+    def self.component_to_ntail_module(component)
+      # this mimicks the ActiveSupport::Inflector.constantize() method in Rails...
+      NginxTail.const_get(self.component_to_module_name(component))
+    end
+
+    attr_reader :raw_line
+    attr_reader :parsable
 
-    attr_accessor :remote_address
-    attr_accessor :remote_user
-    attr_accessor :time_local
-    attr_accessor :request
-    attr_accessor :status
-    attr_accessor :body_bytes_sent
-    attr_accessor :http_referer
-    attr_accessor :http_user_agent
-    attr_accessor :proxy_addresses
-  
-    attr_accessor :http_method
-    attr_accessor :uri
-    attr_accessor :http_version
+    COMPONENTS = [
+      :remote_addr,
+      :remote_user,
+      :time_local,
+      :request,
+      :status,
+      :body_bytes_sent,
+      :http_referer,
+      :http_user_agent,
+      :proxy_addresses,
+    ]
+    
+    COMPONENTS.each do |symbol|
+      attr_reader symbol
+      include component_to_ntail_module(symbol)
+    end
+
+    include KnownIpAddresses # module to identify known IP addresses
+    include LocalIpAddresses # module to identify local IP addresses
+
+    SUBCOMPONENTS = [
+      :http_method,
+      :uri,
+      :http_version,
+    ]
+    
+    SUBCOMPONENTS.each do |symbol|
+      attr_reader symbol
+      include component_to_ntail_module(symbol)
+    end
 
     #
     # http://wiki.nginx.org/NginxHttpLogModule#log_format - we currently only support the default "combined" log format...
@@ -42,7 +60,7 @@ module NginxTail
 
     def initialize(line)
       @parsable = if NGINX_LOG_PATTERN.match(@raw_line = line)
-        @remote_address, @remote_user, @time_local, @request, @status, @body_bytes_sent, @http_referer, @http_user_agent, @proxy_addresses = $~.captures
+        @remote_addr, @remote_user, @time_local, @request, @status, @body_bytes_sent, @http_referer, @http_user_agent, @proxy_addresses = $~.captures
         if NGINX_REQUEST_PATTERN.match(@request)
           # counter example (ie. HTTP request that cannot by parsed)
           # 91.203.96.51 - - [21/Dec/2010:05:26:53 +0000] "-" 400 0 "-" "-"
@@ -57,6 +75,8 @@ module NginxTail
       end
     end
     
+    alias_method :remote_address, :remote_addr # a non-abbreviated alias, for convenience and readability...
+    
     # for now, until we make it fancier...
     def method_missing(method, *params)
       raw_line.send method, *params
@@ -72,52 +92,50 @@ module NginxTail
       else
         :default
       end
-      "%s - %#{Sickill::Rainbow.enabled ? 15 + 9 : 15}s - %s - %s - %s" % [
-        to_date.strftime("%Y-%m-%d %X").foreground(color),
+      "%s - %#{Sickill::Rainbow.enabled ? 15 + 9 : 15}s - %s - %s - %s - %s" % [
+        to_date_s.foreground(color),
         remote_address.foreground(color),
         status.foreground(color),
-        (uri || "-").foreground(color),
-        to_agent_s.foreground(color)
+        to_request_s.foreground(color),
+        to_agent_s.foreground(color),
+        to_referer_s.foreground(color).inverse
       ]
     end
-
-    COMPONENTS = [
-    
-      :remote_address,
-      :remote_user,
-      :time_local,
-      :request,
-      :status,
-      :body_bytes_sent,
-      :http_referer,
-      :http_user_agent,
-      :proxy_addresses,
-
-    ]
-
-    SUBCOMPONENTS = [
-    
-      :http_method,
-      :uri,
-      :http_version,
-    
-    ]
   
     CONVERSIONS = [
   
       :to_date,
+      :to_date_s,
+      
       :to_agent,
+      :to_agent_s,
 
       :to_host_name,
-      :to_country,
-      :to_city,
+      :to_refering_website,
+      
+      :to_country_s,
+      :to_city_s,
     
     ]
 
-    def self.log_subcomponent?(subcomponent) SUBCOMPONENTS.include?(subcomponent) ; end # TODO replace with some clever meta-programming...
-    def self.log_component?(component) COMPONENTS.include?(component) ; end             # TODO replace with some clever meta-programming...
-    def self.log_conversion?(conversion) CONVERSIONS.include?(conversion) ; end         # TODO replace with some clever meta-programming...
-    def self.log_directive?(directive) (directive == :full) or log_conversion?(directive) or log_component?(directive) or log_subcomponent?(directive) ; end
+    def self.log_subcomponent?(subcomponent)
+      # TODO replace with some clever meta-programming...
+      SUBCOMPONENTS.include?(subcomponent)
+    end
+    
+    def self.log_component?(component)
+      # TODO replace with some clever meta-programming...
+      COMPONENTS.include?(component)
+    end
+    
+    def self.log_conversion?(conversion)
+      # TODO replace with some clever meta-programming...
+      CONVERSIONS.include?(conversion)
+    end
+    
+    def self.log_directive?(directive)
+      (directive == :full) or log_conversion?(directive) or log_component?(directive) or log_subcomponent?(directive)
+    end
 
     #
     # extraction filters for log line components
@@ -165,148 +183,6 @@ module NginxTail
     def self.valid_referer?(referer) true ; end
     def self.valid_user_agent?(user_agent) true ; end
 
-    #
-    # conversion of log line components
-    #
-
-    # >> DateTime.strptime("13/Apr/2010:04:45:51 +0100", '%d/%b/%Y:%T %z').to_s
-    # => "2010-04-13T04:45:51+01:00"
-    # >> DateTime.strptime("13/Apr/2010:04:45:51 +0100", '%d/%b/%Y:%H:%M:%S %z').to_s
-    # => "2010-04-13T04:45:51+01:00"
-    # >> _
-
-    def to_date() DateTime.strptime(self.time_local, '%d/%b/%Y:%T %z') ; end
-
-    class SearchBot < Agent
-      attr_accessor :name
-      attr_accessor :os
-      def initialize(string)
-        super string
-        @name = self.class.name_for_user_agent(string)
-        @os = self.class.os_for_user_agent(string)
-      end
-      def self.name_for_user_agent string
-        case string
-          when  GOOGLE_BOT then "googlebot"
-          when     MSN_BOT then "msnbot"
-          when   YAHOO_BOT then "yahoo_slurp"
-          when   ALEXA_BOT then "ia_archiver"
-          when PINGDOM_BOT then "pingdom_bot"
-          when  YANDEX_BOT then "yandex_bot"
-          else super(string)
-        end
-      end
-      def self.os_for_user_agent string
-        case string
-          when  GOOGLE_BOT then "google.com"
-          when     MSN_BOT then "msn.com"
-          when   YAHOO_BOT then "yahoo.com"
-          when   ALEXA_BOT then "alexa.com"
-          when PINGDOM_BOT then "pingdom.com"
-          when  YANDEX_BOT then "yandex.com"
-          else super(string)
-        end
-      end
-    end
-
-    def to_agent() 
-      if known_search_bot?
-        SearchBot.new(self.http_user_agent)
-      else
-        Agent.new(self.http_user_agent)
-      end
-    end
-    
-    def to_agent_s()
-      agent = self.to_agent ; "(%s, %s)" % [agent.name, agent.os]
-    end
-
-    def to_host_name()
-      Socket::getaddrinfo(self.remote_address,nil)[0][2]
-    end
-    
-    if defined? GeoIP # ie. if the optional GeoIP gem is installed
-      
-      if File.exists?('/usr/share/GeoIP/GeoIP.dat')
-        def to_country()
-          record = GeoIP.new('/usr/share/GeoIP/GeoIP.dat').country(self.remote_address) ; record ? record[5] : 'N/A'
-        end
-      end
-      
-      if File.exists?('/usr/share/GeoIP/GeoIPCity.dat')
-        def to_city()
-          record = GeoIP.new('/usr/share/GeoIP/GeoIPCity.dat').city(self.remote_address) ; record ? record[7] : 'N/A'
-        end
-      end
-      
-    end
-
-    #
-    # downstream proxy servers
-    #
-
-    PROXY_IP_ADDRESSES = %w{
-      192.168.0.2
-      192.168.0.3
-      192.168.0.4
-    }
-
-    def self.proxy_ip_address?(remote_address) PROXY_IP_ADDRESSES.include?(remote_address) ; end
-    def      proxy_ip_address?() self.class.proxy_ip_address?(self.remote_address) ; end
-
-    #
-    # known IP addresses, for filtering purposes
-    #
-  
-    OFFICE_IP_ADDRESSES = %w{
-    }
-
-    def self.office_ip_address?(remote_address) OFFICE_IP_ADDRESSES.include?(remote_address) ; end
-    def      office_ip_address?() self.class.office_ip_address?(self.remote_address) ; end
-
-    #
-    # Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
-    # Googlebot-Image/1.0
-    # msnbot/2.0b (+http://search.msn.com/msnbot.htm)
-    # msnbot/2.0b (+http://search.msn.com/msnbot.htm).
-    # msnbot/2.0b (+http://search.msn.com/msnbot.htm)._
-    # Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)
-    # Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
-    # ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
-    # Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
-    #
-  
-    KNOWN_SEARCH_BOTS = [
-       GOOGLE_BOT = Regexp.compile('Googlebot.*\/'),
-          MSN_BOT = Regexp.compile('msnbot\/'),
-        YAHOO_BOT = Regexp.compile('Yahoo! Slurp\/?'),
-      PINGDOM_BOT = Regexp.compile('Pingdom.com_bot_version_'),
-        ALEXA_BOT = Regexp.compile('ia_archiver'),
-       YANDEX_BOT = Regexp.compile('YandexBot\/'),
-      nil
-    ].compact!
-
-    def self.known_search_bot?(user_agent) !KNOWN_SEARCH_BOTS.detect { |bot| bot.match(user_agent) }.nil? end 
-    def      known_search_bot?() self.class.known_search_bot?(self.http_user_agent) ; end
-
-    #
-    # mainly to easily identify external referers, for filtering purposes
-    #
-
-    INTERNAL_REFERERS = [
-      Regexp.compile('^http://(www\.)?MY_WEBSITE_NAME\.com'),
-      Regexp.compile('^-$'),
-    ]
-
-    def self.internal_referer?(http_referer) !INTERNAL_REFERERS.detect { |referer| referer.match(http_referer) }.nil? end
-    def      internal_referer?() self.class.internal_referer?(self.http_referer) ; end
-
-    def self.external_referer?(http_referer) !self.internal_referer?(http_referer) ; end
-    def      external_referer?() self.class.external_referer?(self.http_referer) ; end
-
-    def self.authenticated_user?(remote_user) remote_user and remote_user != "-" ; end
-    def      authenticated_user?() self.class.authenticated_user?(self.remote_user) ; end
-
     #
     # "GET /xd_receiver.html HTTP/1.1"
     # "GET /crossdomain.xml HTTP/1.1"
@@ -352,28 +228,5 @@ module NginxTail
     def self.static_request?(request) !STATIC_REQUESTS.detect { |static_request_regexp| request.match(static_request_regexp) }.nil? end
     def      static_request?()        self.class.static_request?(self.request) ; end
 
-    NGINX_MAGIC_STATUS = '499'   # ex-standard HTTP response code specific to nginx, in addition to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
-    UNPROCESSABLE_ENTITY = '422' # not supported by 'net/http' (Net::HTTPResponse::CODE_TO_OBJ["422"] == nil), see also: http://www.ruby-forum.com/topic/98002 
-
-    # Informational 1xx
-    def self.information_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPInformation ; end
-    def      information_status?()       self.class.information_status?(self.status) ; end
-
-    # Successful 2xx
-    def self.success_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPSuccess ; end
-    def      success_status?()       self.class.success_status?(self.status) ; end
-
-    # Redirection 3xx
-    def self.redirect_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPRedirection ; end
-    def      redirect_status?()       self.class.redirect_status?(self.status) ; end
-
-    # Client Error 4xx
-    def self.client_error_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPClientError ; end
-    def      client_error_status?()       self.class.client_error_status?(self.status) ; end
-
-    # Internal Server Error 5xx
-    def self.server_error_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPServerError ; end
-    def      server_error_status?()       self.class.server_error_status?(self.status) ; end
-  
   end # class LogLine
 end # module NginxTail