ntail 0.0.6 → 0.0.7

data/lib/ntail/proxy_addresses.rb

@@ -0,0 +1,14 @@
+module NginxTail
+  module ProxyAddresses
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'proxy_addresses'" unless base.instance_methods.include? 'proxy_addresses'
+        
+      end
+    end
+        
+  end
+end

data/lib/ntail/remote_addr.rb

@@ -0,0 +1,56 @@
+require 'socket'
+
+begin
+  require 'geoip'
+rescue LoadError
+  # NOOP (optional dependency)
+end
+
+module NginxTail
+  module RemoteAddr
+
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        def self.to_host_s(remote_addr)
+          Socket::getaddrinfo(remote_addr, nil)[0][2]
+        end
+
+        def self.to_country_s(remote_addr)
+          record = if defined? GeoIP # ie. if the optional GeoIP gem is installed
+            if File.exists?('/usr/share/GeoIP/GeoIP.dat') # ie. if the GeoIP country database is installed
+              GeoIP.new('/usr/share/GeoIP/GeoIP.dat').country(remote_addr)
+            end
+          end
+          record ? record[5] : 'N/A'
+        end
+      
+        def self.to_city_s(remote_addr)
+          record = if defined? GeoIP # ie. if the optional GeoIP gem is installed
+            if File.exists?('/usr/share/GeoIP/GeoIPCity.dat') # ie. if the GeoIP city database is installed
+              GeoIP.new('/usr/share/GeoIP/GeoIPCity.dat').city(remote_addr)
+            end
+          end
+          record ? record[7] : 'N/A'
+        end
+        
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'remote_addr'" unless base.instance_methods.include? 'remote_addr'
+
+      end
+    end
+
+    def to_host_s()
+      self.class.to_host_s(self.remote_addr)
+    end
+
+    def to_country_s()
+      self.class.to_country_s(self.remote_addr)
+    end
+
+    def to_city_s()
+      self.class.to_city_s(self.remote_addr)
+    end
+    
+  end
+end

data/lib/ntail/remote_user.rb

@@ -0,0 +1,63 @@
+module NginxTail
+  module RemoteUser
+
+    #
+    # to easily identify remote and authenticated users, for filtering and formatting purposes
+    #
+    # e.g. add all employees as authenticated remote users (from your webserver's .htaccess file)
+    #
+
+    UNKNOWN_REMOTE_USER = "-".freeze # the 'default' nginx value for the $remote_user variable
+
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        @@authenticated_users = []
+
+        # mainly (solely?) for testing purposes...
+        def self.reset_authenticated_users
+          while !@@authenticated_users.empty? ; @@authenticated_users.pop ; end
+        end
+        
+        # mainly (solely?) for testing purposes...
+        def self.authenticated_users
+          @@authenticated_users.dup
+        end
+        
+        def self.add_authenticated_user(authenticated_user)
+          raise "Cannot add unkown remote user" if self.unknown_remote_user? authenticated_user
+          (@@authenticated_users << authenticated_user).uniq!
+        end
+
+        def self.unknown_remote_user?(remote_user)
+          remote_user == UNKNOWN_REMOTE_USER
+        end
+
+        def self.remote_user?(remote_user)
+          !self.unknown_remote_user?(remote_user)
+        end
+        
+        def self.authenticated_user?(remote_user)
+          self.remote_user?(remote_user) && @@authenticated_users.include?(remote_user)
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'remote_user'" unless base.instance_methods.include? 'remote_user'
+
+      end
+    end
+
+    def unknown_remote_user?
+      self.class.unknown_remote_user?(self.remote_user)
+    end
+
+    def remote_user?
+      self.class.remote_user?(self.remote_user)
+    end
+    
+    def authenticated_user?
+      self.class.authenticated_user?(self.remote_user)
+    end
+
+  end
+end

data/lib/ntail/request.rb

@@ -0,0 +1,33 @@
+module NginxTail
+  module Request
+    
+    UNKNOWN_REQUEST = "-".freeze # the 'default' nginx value for $request variable
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        def self.unknown_request?(request)
+          request == UNKNOWN_REQUEST
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'request'" unless base.instance_methods.include? 'request'
+        
+      end
+    end
+
+    def unknown_request?
+      self.class.unknown_request?(self.request)
+    end
+
+    def to_request_s
+      if self.unknown_request?
+        self.request
+      else
+        # note: we exclude the HTTP version info...
+        "%s %s" % [self.to_http_method_s, self.to_uri_s]
+      end
+    end
+        
+  end
+end

data/lib/ntail/status.rb

@@ -0,0 +1,68 @@
+module NginxTail
+  module Status
+    
+    NGINX_MAGIC_STATUS = '499'   # ex-standard HTTP response code specific to nginx, in addition to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        # Informational 1xx
+        def self.information_status?(status)
+          # (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPInformation
+          (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_CLASS_TO_OBJ[(status.to_i / 100).to_s] == Net::HTTPInformation
+        end
+        
+        # Successful 2xx
+        def self.success_status?(status)
+          # (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPSuccess
+          (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_CLASS_TO_OBJ[(status.to_i / 100).to_s] == Net::HTTPSuccess
+        end
+        
+        # Redirection 3xx
+        def self.redirect_status?(status)
+          # (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPRedirection
+          (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_CLASS_TO_OBJ[(status.to_i / 100).to_s] == Net::HTTPRedirection
+        end
+        
+        # Client Error 4xx
+        def self.client_error_status?(status)
+          # (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPClientError
+          (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_CLASS_TO_OBJ[(status.to_i / 100).to_s] == Net::HTTPClientError
+        end
+        
+        # Internal Server Error 5xx
+        def self.server_error_status?(status)
+          # (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPServerError
+          (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_CLASS_TO_OBJ[(status.to_i / 100).to_s] == Net::HTTPServerError
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'status'" unless base.instance_methods.include? 'status'
+        
+      end
+    end
+
+    def information_status?
+      self.class.information_status?(self.status)
+    end
+
+    def success_status?
+      self.class.success_status?(self.status)
+    end
+
+    def redirect_status?
+      self.class.redirect_status?(self.status)
+    end
+
+    def client_error_status?
+      self.class.client_error_status?(self.status)
+    end
+
+    def server_error_status?
+      self.class.server_error_status?(self.status)
+    end
+        
+  end
+end
+
+

data/lib/ntail/time_local.rb

@@ -0,0 +1,38 @@
+require 'date'
+
+module NginxTail
+  module TimeLocal
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        # >> DateTime.strptime("13/Apr/2010:04:45:51 +0100", '%d/%b/%Y:%T %z').to_s
+        # => "2010-04-13T04:45:51+01:00"
+        # >> DateTime.strptime("13/Apr/2010:04:45:51 +0100", '%d/%b/%Y:%H:%M:%S %z').to_s
+        # => "2010-04-13T04:45:51+01:00"
+        # >> _
+
+        def self.to_date(time_local)
+          DateTime.strptime(time_local, '%d/%b/%Y:%T %z')
+        end
+        
+        def self.to_date_s(time_local, format = "%Y-%m-%d %X")
+          self.to_date(time_local).strftime(format)
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'time_local'" unless base.instance_methods.include? 'time_local'
+        
+      end
+    end
+    
+    def to_date
+      self.class.to_date(self.time_local)
+    end
+
+    def to_date_s
+      self.class.to_date_s(self.time_local)
+    end
+    
+  end
+end

data/lib/ntail/uri.rb

@@ -0,0 +1,22 @@
+module NginxTail
+  module Uri
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        def self.to_uri_s(uri)
+          uri || "-" # will be nil if $request == "-" (ie. "dodgy" HTTP requests)
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'uri'" unless base.instance_methods.include? 'uri'
+        
+      end
+    end
+    
+    def to_uri_s
+      self.class.to_uri_s(self.uri)
+    end
+    
+  end
+end

data/ntail.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ntail}
-  s.version = "0.0.6"
+  s.version = "0.0.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Peter Vandenberk"]
-  s.date = %q{2010-12-21}
+  s.date = %q{2011-01-06}
   s.default_executable = %q{ntail}
   s.description = %q{A tail(1)-like utility for nginx log files. It supports parsing, filtering and formatting individual log lines.}
   s.email = %q{pvandenberk@mac.com}
@@ -29,9 +29,34 @@ Gem::Specification.new do |s|
     "bin/ntail",
     "lib/ntail.rb",
     "lib/ntail/application.rb",
+    "lib/ntail/body_bytes_sent.rb",
+    "lib/ntail/http_method.rb",
+    "lib/ntail/http_referer.rb",
+    "lib/ntail/http_user_agent.rb",
+    "lib/ntail/http_version.rb",
+    "lib/ntail/known_ip_addresses.rb",
+    "lib/ntail/local_ip_addresses.rb",
     "lib/ntail/log_line.rb",
+    "lib/ntail/proxy_addresses.rb",
+    "lib/ntail/remote_addr.rb",
+    "lib/ntail/remote_user.rb",
+    "lib/ntail/request.rb",
+    "lib/ntail/status.rb",
+    "lib/ntail/time_local.rb",
+    "lib/ntail/uri.rb",
     "ntail.gemspec",
     "test/helper.rb",
+    "test/ntail/test_http_method.rb",
+    "test/ntail/test_http_referer.rb",
+    "test/ntail/test_http_user_agent.rb",
+    "test/ntail/test_known_ip_addresses.rb",
+    "test/ntail/test_local_ip_addresses.rb",
+    "test/ntail/test_log_line.rb",
+    "test/ntail/test_remote_addr.rb",
+    "test/ntail/test_remote_user.rb",
+    "test/ntail/test_request.rb",
+    "test/ntail/test_status.rb",
+    "test/ntail/test_time_local.rb",
     "test/test_ntail.rb"
   ]
   s.homepage = %q{http://github.com/pvdb/ntail}
@@ -41,6 +66,17 @@ Gem::Specification.new do |s|
   s.summary = %q{A tail(1)-like utility for nginx log files}
   s.test_files = [
     "test/helper.rb",
+    "test/ntail/test_http_method.rb",
+    "test/ntail/test_http_referer.rb",
+    "test/ntail/test_http_user_agent.rb",
+    "test/ntail/test_known_ip_addresses.rb",
+    "test/ntail/test_local_ip_addresses.rb",
+    "test/ntail/test_log_line.rb",
+    "test/ntail/test_remote_addr.rb",
+    "test/ntail/test_remote_user.rb",
+    "test/ntail/test_request.rb",
+    "test/ntail/test_status.rb",
+    "test/ntail/test_time_local.rb",
     "test/test_ntail.rb"
   ]
 
@@ -55,6 +91,7 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<geoip>, [">= 0"])
     else
       s.add_dependency(%q<rainbow>, [">= 0"])
       s.add_dependency(%q<user-agent>, [">= 0"])
@@ -62,6 +99,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<geoip>, [">= 0"])
     end
   else
     s.add_dependency(%q<rainbow>, [">= 0"])
@@ -70,6 +108,7 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
     s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<geoip>, [">= 0"])
   end
 end
 

data/test/helper.rb

@@ -15,4 +15,77 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'ntail'
 
 class Test::Unit::TestCase
+
+  def random_ip_address
+    ((1..4).map { Kernel.rand(256) }).join('.')
+  end
+  
+  def local_ip_address
+    # http://en.wikipedia.org/wiki/IP_address#IPv4_private_addresses
+    (['192', '168'] + (1..2).map { Kernel.rand(256) }).join('.')
+  end
+
+  # 
+  # http://wiki.nginx.org/NginxHttpLogModule#log_format
+  #
+  # There is a predefined log format called "combined":
+  # 
+  # log_format combined '$remote_addr - $remote_user [$time_local] '
+  #                     '"$request" $status $body_bytes_sent '
+  #                     '"$http_referer" "$http_user_agent"';
+  # 
+
+  LOG_FORMAT_COMBINED = '%s - %s [%s] ' \
+                        '"%s" %d %d ' \
+                        '"%s" "%s"'
+                        
+  DEFAULT_VALUES = {
+    :remote_addr => '72.46.130.42',
+    :remote_user => '-',
+    :time_local => '01/Jan/2010:04:00:29 +0000',
+    # :request => 'GET /index.html HTTP/1.1',
+      :http_method => 'GET',
+      :uri => '/index.html',
+      :http_version => 'HTTP/1.1',
+    :status => 200,
+    :body_bytes_sent => 6918,
+    :http_referer => 'http://www.google.com/search?q=example',
+    :http_user_agent => 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10'
+  }
+  
+  REQUEST_FORMAT = '%s %s %s'
+  
+  def random_raw_line(options = {})
+    options = DEFAULT_VALUES.merge options
+    options[:request] ||= REQUEST_FORMAT % [
+      options[:http_method],
+      options[:uri],
+      options[:http_version]
+    ]
+    LOG_FORMAT_COMBINED % [
+      options[:remote_addr],
+      options[:remote_user],
+      options[:time_local],
+      options[:request],
+      options[:status],
+      options[:body_bytes_sent],
+      options[:http_referer],
+      options[:http_user_agent],
+    ]
+  end
+  
+  def random_log_line(options = {})
+    NginxTail::LogLine.new(random_raw_line(options))
+  end
+  
+  def bad_request_raw_line
+    # a "bad request", resulting in a 400 status, is logged by nginx as follows:
+    # 121.8.101.138 - - [28/Dec/2010:23:50:58 +0000] "-" 400 0 "-" "-"
+    '121.8.101.138 - - [28/Dec/2010:23:50:58 +0000] "-" 400 0 "-" "-"'
+  end
+  
+  def bad_request_log_line
+    NginxTail::LogLine.new(bad_request_raw_line)
+  end
+  
 end