ntail 0.0.6 → 0.0.7

data/Gemfile

@@ -12,4 +12,5 @@ group :development do
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.5.1"
   gem "rcov", ">= 0"
+  gem "geoip", ">= 0"
 end

data/Gemfile.lock

@@ -1,6 +1,7 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    geoip (0.8.7)
     git (1.2.5)
     jeweler (1.5.1)
       bundler (~> 1.0.0)
@@ -17,6 +18,7 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0.0)
+  geoip
   jeweler (~> 1.5.1)
   rainbow
   rcov

data/README.rdoc

@@ -49,7 +49,7 @@ A tail(1)-like utility for nginx log files that supports parsing, filtering and
 
 * print out GeoIP country and city information for each HTTP request <em>(depends on the optional <tt>geoip</tt> gem)</em>
 
-    > ntail -e '{ |line| puts [line.to_country, line.to_city].join("\t") }' /var/log/nginx/access.log
+    > ntail -e '{ |line| puts [line.to_country_s, line.to_city_s].join("\t") }' /var/log/nginx/access.log
     United States   Los Angeles
     United States   Houston
     Germany         Berlin

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.0.7

data/lib/ntail.rb

@@ -1,5 +1,28 @@
 NTAIL_NAME = 'ntail'
 NTAIL_VERSION = '0.0.1'
 
+# so-called components...
+require 'ntail/remote_addr'
+require 'ntail/remote_user'
+require 'ntail/time_local'
+require 'ntail/request'
+require 'ntail/status'
+require 'ntail/body_bytes_sent'
+require 'ntail/http_referer'
+require 'ntail/http_user_agent'
+require 'ntail/proxy_addresses'
+
+# so-called sub-components...
+require 'ntail/http_method'
+require 'ntail/uri'
+require 'ntail/http_version'
+
+# additional utility functions...
+require 'ntail/known_ip_addresses'
+require 'ntail/local_ip_addresses'
+
+# the core classes...
 require 'ntail/log_line'
-require 'ntail/application'
+require 'ntail/application'
+
+# That's all, Folks!

data/lib/ntail/application.rb

@@ -12,17 +12,32 @@ module NginxTail
     def self.ntail_options
       # shamelessly copied from lib/rake.rb (rake gem)
       [
-        ['--verbose', '--v', "Run verbosely (log messages to STDERR).",
+        ['--verbose', '-v', "Run verbosely (log messages to STDERR).",
           lambda { |value|
             self.options.verbose = true
           }
         ],
+        ['--dry-run', '-n', "Dry-run: process files, but don't actually parse the lines",
+          lambda { |value|
+            self.options.dry_run = true
+          }
+        ],
+        ['--parse-only', '-p', "Parse only: parse all lines, but don't actually process them",
+          lambda { |value|
+            self.options.parse_only = true
+          }
+        ],
         ['--version', '-V', "Display the program version.",
           lambda { |value|
             puts "#{NTAIL_NAME}, version #{NTAIL_VERSION}"
             self.options.running = false
           }
         ],
+        ['--line-number', '-l LINE_NUMBER', "Only process the line with the given line number",
+          lambda { |value|
+            self.options.line_number = value.to_i
+          }
+        ],
         ['--filter',  '-f CODE', "Ruby code block for filtering (parsed) lines - needs to return true or false.",
           lambda { |value|
             self.options.filter = eval "Proc.new #{value}"
@@ -39,6 +54,7 @@ module NginxTail
     def self.parse_options
       
       # application defaults...
+      self.options.interrupted = false
       self.options.running = true
       self.options.exit = 0
       
@@ -62,47 +78,63 @@ module NginxTail
       
       ['TERM', 'INT'].each do |signal|
         Signal.trap(signal) do
-          self.options.running = false ; puts
+          self.options.running = false ; self.options.interrupted = true
+          $stdin.close if ARGF.file == $stdin # ie. reading from STDIN
         end
       end
       
-      lines_read = lines_processed = lines_ignored = parsable_lines = unparsable_lines = 0
+      files_read = lines_read = lines_processed = lines_ignored = parsable_lines = unparsable_lines = 0
       
       while self.options.running and ARGF.gets
+        if ARGF.file.lineno == 1
+          files_read += 1
+          if self.options.verbose
+            $stderr.puts "[INFO] now processing file #{ARGF.filename}"
+          end
+        end
         raw_line = $_.chomp ; lines_read += 1
-        begin
-          log_line = NginxTail::LogLine.new(raw_line)
-          if log_line.parsable
-            parsable_lines += 1
-            if !self.options.filter || self.options.filter.call(log_line)
-              lines_processed += 1
-              if self.options.code
-                self.options.code.call(log_line)
+        unless self.options.dry_run
+          if !self.options.line_number or self.options.line_number == ARGF.lineno
+            begin
+              log_line = NginxTail::LogLine.new(raw_line)
+              if log_line.parsable
+                parsable_lines += 1
+                unless self.options.parse_only
+                  if !self.options.filter || self.options.filter.call(log_line)
+                    lines_processed += 1
+                    if self.options.code
+                      self.options.code.call(log_line)
+                    else
+                      puts log_line
+                    end
+                  else
+                    lines_ignored += 1
+                    if self.options.verbose
+                      $stderr.puts "[WARNING] ignoring line ##{lines_read}"
+                    end
+                  end
+                end
               else
-                puts log_line
+                unparsable_lines += 1
+                if self.options.verbose
+                  $stderr.puts "[ERROR] cannot parse '#{raw_line}'"
+                end
               end
-            else
-              lines_ignored += 1
-              if self.options.verbose
-                $stderr.puts "[WARNING] ignoring line ##{lines_read}"
-              end
-            end
-          else
-            unparsable_lines += 1
-            if self.options.verbose
-              $stderr.puts "[ERROR] cannot parse '#{raw_line}'"
+            rescue
+              $stderr.puts "[ERROR] processing line #{ARGF.file.lineno} of file #{ARGF.filename} resulted in #{$!.message}"
+              $stderr.puts "[ERROR] " + raw_line
+              self.options.exit = -1
+              self.options.running = false
+              raise $! # TODO if the "re-raise exceptions" option has been set...
             end
           end
-        rescue
-          $stderr.puts "[ERROR] processing line #{lines_read} resulted in #{$!.message}"
-          $stderr.puts "[ERROR] " + raw_line
-          self.options.exit = -1
-          self.options.running = false
         end
       end
       
       if self.options.verbose
-        $stderr.puts "[INFO] read #{lines_read} lines"
+        $stderr.puts if self.options.interrupted
+        $stderr.print "[INFO] read #{lines_read} lines in #{files_read} files"
+        $stderr.print " (interrupted)" if self.options.interrupted ; $stderr.puts
         $stderr.puts "[INFO] #{parsable_lines} parsable lines, #{unparsable_lines} unparsable lines"
         $stderr.puts "[INFO] processed #{lines_processed} lines, ignored #{lines_ignored} lines"
       end

data/lib/ntail/body_bytes_sent.rb

@@ -0,0 +1,14 @@
+module NginxTail
+  module BodyBytesSent
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'body_bytes_sent'" unless base.instance_methods.include? 'body_bytes_sent'
+        
+      end
+    end
+        
+  end
+end

data/lib/ntail/http_method.rb

@@ -0,0 +1,34 @@
+module NginxTail
+  module HttpMethod
+    
+    #
+    # http://www.ietf.org/rfc/rfc2616.txt - "section 5.1.1 Method"
+    # 
+    # OPTIONS GET HEAD POST PUT DELETE TRACE CONNECT 
+    #
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+        
+        def self.to_http_method_s(http_method)
+          (http_method ||= "").upcase!  # will be nil if $request == "-" (ie. "dodgy" HTTP requests)
+          case http_method
+          when "POST", "PUT", "DELETE"
+            http_method.inverse # if Sickill::Rainbow.enabled...
+          else
+            http_method
+          end
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'http_method'" unless base.instance_methods.include? 'http_method'
+        
+      end
+    end
+    
+    def to_http_method_s
+      self.class.to_http_method_s(self.http_method)
+    end
+    
+  end
+end

data/lib/ntail/http_referer.rb

@@ -0,0 +1,81 @@
+require 'uri'
+
+module NginxTail
+  module HttpReferer
+
+    #
+    # to easily identify external referers, for filtering and formatting purposes
+    #
+    # e.g. Regexp.compile('^http(s)?://(www\.)?MY_WEBSITE_NAME\.com')
+    #
+
+    UNKNOWN_REFERER = "-".freeze # the 'default' nginx value for $http_referer variable
+
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        @@internal_referers = []
+
+        # mainly (solely?) for testing purposes...
+        def self.reset_internal_referers()
+          while !@@internal_referers.empty? ; @@internal_referers.pop ; end
+        end
+        
+        # mainly (solely?) for testing purposes...
+        def self.internal_referers()
+          @@internal_referers.dup
+        end
+        
+        def self.add_internal_referer(internal_referer)
+          raise "Cannot add unkown HTTP referer" if self.unknown_referer? internal_referer
+          (@@internal_referers << internal_referer).uniq!
+        end
+
+        def self.unknown_referer?(http_referer)
+          http_referer == UNKNOWN_REFERER
+        end
+        
+        def self.internal_referer?(http_referer)
+          !self.unknown_referer?(http_referer) && !@@internal_referers.detect { |referer| referer.match(http_referer) }.nil?
+        end
+        
+        def self.external_referer?(http_referer)
+          !self.unknown_referer?(http_referer) && !self.internal_referer?(http_referer)
+        end
+
+         def self.to_referer_s(http_referer)
+           if self.unknown_referer? http_referer
+             http_referer
+           else begin
+               # try to parse it as a URI, but with default value if un-parsable
+               URI.parse(http_referer).host || http_referer
+            rescue URI::InvalidURIError
+               http_referer
+            end
+          end
+        end
+        
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'http_referer'" unless base.instance_methods.include? 'http_referer'
+
+      end
+    end
+
+    def unknown_referer?
+      self.class.unknown_referer?(self.http_referer)
+    end
+    
+    def internal_referer?
+      self.class.internal_referer?(self.http_referer)
+    end
+    
+    def external_referer?
+      self.class.external_referer?(self.http_referer)
+    end
+    
+    def to_referer_s
+      self.class.to_referer_s(self.http_referer)
+    end
+
+  end
+end

data/lib/ntail/http_user_agent.rb

@@ -0,0 +1,122 @@
+require 'rubygems'
+require 'user-agent'
+
+class Agent
+  
+  def search_bot?
+    false
+  end
+
+end
+
+class SearchBot < Agent
+
+  def search_bot?
+    true
+  end
+  
+  #
+  # Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+  # Googlebot-Image/1.0
+  # msnbot/2.0b (+http://search.msn.com/msnbot.htm)
+  # msnbot/2.0b (+http://search.msn.com/msnbot.htm).
+  # msnbot/2.0b (+http://search.msn.com/msnbot.htm)._
+  # Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)
+  # Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
+  # ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
+  # Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
+  #
+
+  KNOWN_SEARCH_BOTS = [
+     GOOGLE_BOT = Regexp.compile('Googlebot.*\/'),
+        MSN_BOT = Regexp.compile('msnbot\/'),
+      YAHOO_BOT = Regexp.compile('Yahoo! Slurp\/?'),
+    PINGDOM_BOT = Regexp.compile('Pingdom.com_bot_version_'),
+      ALEXA_BOT = Regexp.compile('ia_archiver'),
+     YANDEX_BOT = Regexp.compile('YandexBot\/'),
+  ]
+
+  def self.search_bot?(http_user_agent)
+    !KNOWN_SEARCH_BOTS.detect { |bot| bot.match(http_user_agent) }.nil?
+  end 
+  
+  attr_accessor :name
+  attr_accessor :os
+
+  def initialize(string)
+    super string
+    @name = self.class.name_for_user_agent(string)
+    @os = self.class.os_for_user_agent(string)
+  end
+
+  def self.name_for_user_agent string
+    case string
+      when  GOOGLE_BOT then :googlebot
+      when     MSN_BOT then :msnbot
+      when   YAHOO_BOT then :yahoo_slurp
+      when   ALEXA_BOT then :ia_archiver
+      when PINGDOM_BOT then :pingdom_bot
+      when  YANDEX_BOT then :yandex_bot
+      else super(string)
+    end
+  end
+  
+  def self.os_for_user_agent string
+    case string
+      when  GOOGLE_BOT then :"google.com"
+      when     MSN_BOT then :"msn.com"
+      when   YAHOO_BOT then :"yahoo.com"
+      when   ALEXA_BOT then :"alexa.com"
+      when PINGDOM_BOT then :"pingdom.com"
+      when  YANDEX_BOT then :"yandex.com"
+      else super(string)
+    end
+  end
+  
+end
+
+module NginxTail
+  module HttpUserAgent
+    
+    def self.included(base) # :nodoc:
+      base.class_eval do
+
+        def self.search_bot?(http_user_agent)
+          SearchBot.search_bot?(http_user_agent)
+        end 
+
+        def self.to_agent(http_user_agent)
+          if self.search_bot? http_user_agent
+            SearchBot.new(http_user_agent)
+          else
+            Agent.new(http_user_agent)
+          end
+        end
+
+        def self.to_agent_s(http_user_agent)
+          agent = self.to_agent http_user_agent
+          "(%s, %s)" % [agent.name, agent.os]
+        end
+
+        # this ensures the below module methods actually make sense...
+        raise "Class #{base.name} should implement instance method 'http_user_agent'" unless base.instance_methods.include? 'http_user_agent'
+        
+      end
+    end
+    
+    def search_bot?
+      self.class.search_bot?(self.http_user_agent)
+    end
+    
+    def to_agent
+      self.class.to_agent(self.http_user_agent)
+    end
+    
+    def to_agent_s
+      self.class.to_agent_s(self.http_user_agent)
+    end
+    
+  end
+end
+
+