ntail 0.0.1

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,15 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.1"
+  gem "rcov", ">= 0"
+  gem "rainbow", ">= 0"
+  gem "user-agent", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,24 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.5.1)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    rainbow (1.1)
+    rake (0.8.7)
+    rcov (0.9.9)
+    shoulda (2.11.3)
+    user-agent (1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.5.1)
+  rainbow
+  rcov
+  shoulda
+  user-agent

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Peter Vandenberk
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,90 @@
+= ntail
+
+A parsing tail(1)-like utility for nginx log files
+
+  > gem install ntail
+
+== Examples
+
+* read from STDIN and print each line to STDOUT <em>(stop with ^D)</em>
+
+    > ntail
+
+* process an nginx log file and print each line to STDOUT
+
+    > ntail /var/log/nginx/access.log
+
+* tail an "active" nginx log file and print each new line to STDOUT <em>(stop with ^C)</em>
+
+    > tail -f /var/log/nginx/access.log | ntail
+
+* tail STDIN and print out the length of each line <em>(to illustrate -e option)</em>
+
+    > ntail -e '{ |line| puts line.size }'
+
+* tail STDIN but only print out non-empty lines <em>(to illustrate -f option)</em>
+
+    > ntail -f '{ |line| line.size > 0 }'
+
+* the following invocations behave exactly the same <em>(to illustrate -e and -f options)</em>
+
+    > ntail
+    > ntail -f '{ |line| true }' -e '{ |line| puts line }'
+
+* find all HTTP requests that resulted in a '5xx' HTTP error/status code <em>(e.g. Rails 500 errors)</em>
+
+    > gunzip -S .gz -c access.log-20101216.gz | ntail -f '{ |line| line.server_error_status? }'
+
+* generate a summary report of HTTP status codes, for all non-200 HTTP requests
+
+    > ntail -f '{ |line| line.status != "200" }' -e '{ |line| puts line.status }' access.log | sort | uniq -c
+    76 301
+    16 302
+     2 304
+     1 406
+
+* print out GeoIP country and city information for each HTTP request <em>(depends on the optional <tt>geoip</tt> gem)</em>
+
+    > ntail -e '{ |line| puts [line.to_country, line.to_city].join("\t") }' /var/log/nginx/access.log
+    United States   Los Angeles
+    United States   Houston
+    Germany         Berlin
+    United Kingdom  London
+
+* print out the IP address and the corresponding host name for each HTTP request <em>(slows things down considerably, due to <tt>nslookup</tt> call)</em>
+
+    > ntail -e '{ |line| puts [line.remote_address, line.to_host_name].join("\t") }' /var/log/nginx/access.log
+    66.249.72.196   crawl-66-249-72-196.googlebot.com
+    67.192.120.134  s402.pingdom.com
+    75.31.109.144   adsl-75-31-109-144.dsl.irvnca.sbcglobal.net
+
+== TODO
+
+* implement a native <tt>"-f"</tt> option for ntail, similar to that of <tt>tail(1)</tt>
+* implement a <tt>"-i"</tt> option ("ignore exceptions"/"continue processing"), if handling a single line raised an exception
+
+* make <tt>PROXY_IP_ADDRESS</tt> configurable (from command line and/or rc file)
+* make <tt>OFFICE_IP_ADDRESS</tt> configurable (from command line and/or rc file)
+* make <tt>KNOWN_SEARCH_BOTS</tt> configurable (from command line and/or rc file)
+* make <tt>INTERNAL_REFERERS</tt> configurable (from command line and/or rc file)
+* make <tt>AUTOMATED_REQUESTS</tt> configurable (from command line and/or rc file)
+* make <tt>STATIC_REPOS</tt> configurable (from command line and/or rc file)
+
+== Acknowledgements
+
+ntail's parsing feature is inspired by an nginx log parser written by {Richard Taylor (moomerman)}[https://github.com/moomerman]
+
+== Contributing to ntail
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2010 Peter Vandenberk. See LICENSE.txt for further details.
+

data/Rakefile

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "ntail"
+  gem.homepage = "http://github.com/pvdb/ntail"
+  gem.license = "MIT"
+  gem.summary = %Q{A parsing tail(1)-like utility for nginx log files}
+  gem.description = %Q{A parsing tail(1)-like utility for nginx log files}
+  gem.email = "pvandenberk@mac.com"
+  gem.authors = ["Peter Vandenberk"]
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "ntail #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/bin/ntail

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+begin
+  require 'ntail'
+rescue LoadError
+  require 'rubygems'
+  require 'ntail'
+end
+
+exit NginxTail::Application.run!
+
+# That's all, Folks!

data/lib/ntail.rb

@@ -0,0 +1,5 @@
+NTAIL_NAME = 'ntail'
+NTAIL_VERSION = '0.0.1'
+
+require 'ntail/log_line'
+require 'ntail/application'

data/lib/ntail/application.rb

@@ -0,0 +1,115 @@
+require 'ostruct'
+require 'optparse'
+
+module NginxTail
+  class Application
+    
+    def self.options
+      # application options from the command line
+      @@options ||= OpenStruct.new
+    end
+    
+    def self.ntail_options
+      # shamelessly copied from lib/rake.rb (rake gem)
+      [
+        ['--verbose', '--v', "Run verbosely (log messages to STDERR).",
+          lambda { |value|
+            self.options.verbose = true
+          }
+        ],
+        ['--version', '-V', "Display the program version.",
+          lambda { |value|
+            puts "#{NTAIL_NAME}, version #{NTAIL_VERSION}"
+            self.options.running = false
+          }
+        ],
+        ['--filter',  '-f CODE', "Ruby code block for filtering (parsed) lines - needs to return true or false.",
+          lambda { |value|
+            self.options.filter = eval "Proc.new #{value}"
+          }
+        ],
+        ['--execute',  '-e CODE', "Ruby code block for processing each (parsed) line.",
+          lambda { |value|
+            self.options.code = eval "Proc.new #{value}"
+          }
+        ],
+      ]
+    end
+
+    def self.parse_options
+      
+      # application defaults...
+      self.options.running = true
+      self.options.exit = 0
+      
+      OptionParser.new do |opts|
+        opts.banner = "ntail {options} {file(s)} ..."
+        opts.separator ""
+        opts.separator "Options are ..."
+
+        opts.on_tail("-h", "--help", "-H", "Display this help message.") do
+          puts opts
+          self.options.running = false
+        end
+
+        self.ntail_options.each { |args| opts.on(*args) }
+      end.parse!
+    end
+    
+    def self.run!
+      
+      self.parse_options
+      
+      ['TERM', 'INT'].each do |signal|
+        Signal.trap(signal) do
+          self.options.running = false ; puts
+        end
+      end
+      
+      lines_read = lines_processed = lines_ignored = parsable_lines = unparsable_lines = 0
+      
+      while self.options.running and ARGF.gets
+        raw_line = $_.chomp ; lines_read += 1
+        begin
+          log_line = NginxTail::LogLine.new(raw_line)
+          if log_line.parsable
+            parsable_lines += 1
+            if !self.options.filter || self.options.filter.call(log_line)
+              lines_processed += 1
+              if self.options.code
+                self.options.code.call(log_line)
+              else
+                puts log_line
+              end
+            else
+              lines_ignored += 1
+              if self.options.verbose
+                $stderr.puts "[WARNING] ignoring line ##{lines_read}"
+              end
+            end
+          else
+            unparsable_lines += 1
+            if self.options.verbose
+              $stderr.puts "[ERROR] cannot parse '#{raw_line}'"
+            end
+          end
+        rescue
+          $stderr.puts "[ERROR] processing line #{lines_read} resulted in #{$!.message}"
+          $stderr.puts "[ERROR] " + raw_line
+          self.options.exit = -1
+          self.options.running = false
+        end
+      end
+      
+      if self.options.verbose
+        $stderr.puts "[INFO] read #{lines_read} lines"
+        $stderr.puts "[INFO] #{parsable_lines} parsable lines, #{unparsable_lines} unparsable lines"
+        $stderr.puts "[INFO] processed #{lines_processed} lines, ignored #{lines_ignored} lines"
+      end
+      
+      return self.options.exit
+      
+    end # def run
+    
+  end
+end

data/lib/ntail/log_line.rb

@@ -0,0 +1,375 @@
+require 'socket'
+require 'net/http'
+
+require 'rubygems'
+require 'rainbow'
+require 'user-agent'
+
+begin
+  require 'geoip'
+rescue
+  # NOOP (optional dependency)
+end
+
+module NginxTail
+  class LogLine
+
+    attr_accessor :raw_line
+    attr_accessor :parsable
+
+    attr_accessor :remote_address
+    attr_accessor :remote_user
+    attr_accessor :time_local
+    attr_accessor :request
+    attr_accessor :status
+    attr_accessor :body_bytes_sent
+    attr_accessor :http_referer
+    attr_accessor :http_user_agent
+    attr_accessor :proxy_addresses
+  
+    attr_accessor :http_method
+    attr_accessor :uri
+    attr_accessor :http_version
+
+    #
+    # http://wiki.nginx.org/NginxHttpLogModule#log_format - we currently only support the default "combined" log format...
+    #
+
+    NGINX_LOG_PATTERN = Regexp.compile(/\A(\S+) - (\S+) \[([^\]]+)\] "([^"]+)" (\S+) (\S+) "([^"]*?)" "([^"]*?)"( "([^"]*?)")?\Z/)
+    NGINX_REQUEST_PATTERN = Regexp.compile(/\A(\S+) (.*?) (\S+)\Z/)
+    NGINX_PROXY_PATTERN = Regexp.compile(/\A "([^"]*)"\Z/)
+
+    def initialize(line)
+      @parsable = if NGINX_LOG_PATTERN.match(@raw_line = line)
+        @remote_address, @remote_user, @time_local, @request, @status, @body_bytes_sent, @http_referer, @http_user_agent, @proxy_addresses = $~.captures
+        if NGINX_REQUEST_PATTERN.match(@request)
+          @http_method, @uri, @http_version = $~.captures
+        end
+        if @proxy_addresses and NGINX_PROXY_PATTERN.match(@proxy_addresses)
+          @proxy_addresses = $~.captures.first.split(/, /)
+        end
+        true
+      else
+        false
+      end
+    end
+    
+    # for now, until we make it fancier...
+    def method_missing(method, *params)
+      raw_line.send method, *params
+    end
+    
+    def to_s()
+      # simple but boring:
+      # raw_line.to_s
+      color = if redirect_status?
+        :yellow
+      elsif !success_status?
+        :red
+      else
+        :default
+      end
+      "%#{Sickill::Rainbow.enabled ? 15 + 9 : 15}s - %s - %s - %s" % [
+        remote_address.foreground(color),
+        status.foreground(color),
+        uri.foreground(color),
+        to_agent_s.foreground(color)
+      ]
+    end
+
+    COMPONENTS = [
+    
+      :remote_address,
+      :remote_user,
+      :time_local,
+      :request,
+      :status,
+      :body_bytes_sent,
+      :http_referer,
+      :http_user_agent,
+      :proxy_addresses,
+
+    ]
+
+    SUBCOMPONENTS = [
+    
+      :http_method,
+      :uri,
+      :http_version,
+    
+    ]
+  
+    CONVERSIONS = [
+  
+      :to_date,
+      :to_agent,
+
+      :to_host_name,
+      :to_country,
+      :to_city,
+    
+    ]
+
+    def self.log_subcomponent?(subcomponent) SUBCOMPONENTS.include?(subcomponent) ; end # TODO replace with some clever meta-programming...
+    def self.log_component?(component) COMPONENTS.include?(component) ; end             # TODO replace with some clever meta-programming...
+    def self.log_conversion?(conversion) CONVERSIONS.include?(conversion) ; end         # TODO replace with some clever meta-programming...
+    def self.log_directive?(directive) (directive == :full) or log_conversion?(directive) or log_component?(directive) or log_subcomponent?(directive) ; end
+
+    #
+    # extraction filters for log line components
+    #
+
+    def self.regexp_for_remote_address(remote_address)
+      Regexp.compile(/^(#{remote_address}) /)
+    end
+
+    def self.regexp_for_request(request)
+      Regexp.compile(/^([^"]+) "([^"]*#{request}[^"]*)" /)
+    end
+
+    def self.regexp_for_status(status)
+      Regexp.compile(/ "([^"]+)" (#{status}) /)
+    end
+
+    def self.regexp_for_http_referer(http_referer)
+      Regexp.compile(/" .* "([^"]*#{http_referer}[^"]*)" "/)
+    end
+
+    def self.regexp_for_http_user_agent(http_user_agent)
+      Regexp.compile(/ "([^"]*#{http_user_agent}[^"]*)"$/)
+    end
+ 
+    #
+    # validation of log line components
+    #
+
+    def self.valid_status?(status)
+      if /\A(\d{1,3})\Z/ =~ status
+        return $~.captures.all? { |i| 100 <= i.to_i and i.to_i < 600 }
+      end
+      return false
+    end
+
+    def self.valid_v4?(addr)
+      if /\A(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\Z/ =~ addr
+        return $~.captures.all? {|i| i.to_i < 256}
+      end
+      return false
+    end
+
+    def self.valid_request?(request) true ; end
+    def self.valid_referer?(referer) true ; end
+    def self.valid_user_agent?(user_agent) true ; end
+
+    #
+    # conversion of log line components
+    #
+
+    # >> DateTime.strptime("13/Apr/2010:04:45:51 +0100", '%d/%b/%Y:%T %z').to_s
+    # => "2010-04-13T04:45:51+01:00"
+    # >> DateTime.strptime("13/Apr/2010:04:45:51 +0100", '%d/%b/%Y:%H:%M:%S %z').to_s
+    # => "2010-04-13T04:45:51+01:00"
+    # >> _
+
+    def to_date() DateTime.strptime(self.time_local, '%d/%b/%Y:%T %z') ; end
+
+    class SearchBot < Agent
+      attr_accessor :name
+      attr_accessor :os
+      def initialize(string)
+        super string
+        @name = self.class.name_for_user_agent(string)
+        @os = self.class.os_for_user_agent(string)
+      end
+      def self.name_for_user_agent string
+        case string
+          when  GOOGLE_BOT then "googlebot"
+          when     MSN_BOT then "msnbot"
+          when   YAHOO_BOT then "yahoo_slurp"
+          when   ALEXA_BOT then "ia_archiver"
+          when PINGDOM_BOT then "pingdom_bot"
+          when  YANDEX_BOT then "yandex_bot"
+          else super(string)
+        end
+      end
+      def self.os_for_user_agent string
+        case string
+          when  GOOGLE_BOT then "google.com"
+          when     MSN_BOT then "msn.com"
+          when   YAHOO_BOT then "yahoo.com"
+          when   ALEXA_BOT then "alexa.com"
+          when PINGDOM_BOT then "pingdom.com"
+          when  YANDEX_BOT then "yandex.com"
+          else super(string)
+        end
+      end
+    end
+
+    def to_agent() 
+      if known_search_bot?
+        SearchBot.new(self.http_user_agent)
+      else
+        Agent.new(self.http_user_agent)
+      end
+    end
+    
+    def to_agent_s()
+      agent = self.to_agent ; "(%s, %s)" % [agent.name, agent.os]
+    end
+
+    def to_host_name()
+      Socket::getaddrinfo(self.remote_address,nil)[0][2]
+    end
+    
+    if defined? GeoIP # ie. if the optional GeoIP gem is installed
+      
+      if File.exists?('/usr/share/GeoIP/GeoIP.dat')
+        def to_country()
+          record = GeoIP.new('/usr/share/GeoIP/GeoIP.dat').country(self.remote_address) ; record ? record[5] : 'N/A'
+        end
+      end
+      
+      if File.exists?('/usr/share/GeoIP/GeoIPCity.dat')
+        def to_city()
+          record = GeoIP.new('/usr/share/GeoIP/GeoIPCity.dat').city(self.remote_address) ; record ? record[7] : 'N/A'
+        end
+      end
+      
+    end
+
+    #
+    # downstream proxy servers
+    #
+
+    PROXY_IP_ADDRESSES = %w{
+      192.168.0.2
+      192.168.0.3
+      192.168.0.4
+    }
+
+    def self.proxy_ip_address?(remote_address) PROXY_IP_ADDRESSES.include?(remote_address) ; end
+    def      proxy_ip_address?() self.class.proxy_ip_address?(self.remote_address) ; end
+
+    #
+    # known IP addresses, for filtering purposes
+    #
+  
+    OFFICE_IP_ADDRESSES = %w{
+    }
+
+    def self.office_ip_address?(remote_address) OFFICE_IP_ADDRESSES.include?(remote_address) ; end
+    def      office_ip_address?() self.class.office_ip_address?(self.remote_address) ; end
+
+    #
+    # Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)
+    # Googlebot-Image/1.0
+    # msnbot/2.0b (+http://search.msn.com/msnbot.htm)
+    # msnbot/2.0b (+http://search.msn.com/msnbot.htm).
+    # msnbot/2.0b (+http://search.msn.com/msnbot.htm)._
+    # Mozilla/5.0 (compatible; Yahoo! Slurp/3.0; http://help.yahoo.com/help/us/ysearch/slurp)
+    # Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)
+    # ia_archiver (+http://www.alexa.com/site/help/webmasters; crawler@alexa.com)
+    # Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
+    #
+  
+    KNOWN_SEARCH_BOTS = [
+       GOOGLE_BOT = Regexp.compile('Googlebot.*\/'),
+          MSN_BOT = Regexp.compile('msnbot\/'),
+        YAHOO_BOT = Regexp.compile('Yahoo! Slurp\/?'),
+      PINGDOM_BOT = Regexp.compile('Pingdom.com_bot_version_'),
+        ALEXA_BOT = Regexp.compile('ia_archiver'),
+       YANDEX_BOT = Regexp.compile('YandexBot\/'),
+      nil
+    ].compact!
+
+    def self.known_search_bot?(user_agent) !KNOWN_SEARCH_BOTS.detect { |bot| bot.match(user_agent) }.nil? end 
+    def      known_search_bot?() self.class.known_search_bot?(self.http_user_agent) ; end
+
+    #
+    # mainly to easily identify external referers, for filtering purposes
+    #
+
+    INTERNAL_REFERERS = [
+      Regexp.compile('^http://(www\.)?MY_WEBSITE_NAME\.com'),
+      Regexp.compile('^-$'),
+    ]
+
+    def self.internal_referer?(http_referer) !INTERNAL_REFERERS.detect { |referer| referer.match(http_referer) }.nil? end
+    def      internal_referer?() self.class.internal_referer?(self.http_referer) ; end
+
+    def self.external_referer?(http_referer) !self.internal_referer?(http_referer) ; end
+    def      external_referer?() self.class.external_referer?(self.http_referer) ; end
+
+    def self.authenticated_user?(remote_user) remote_user and remote_user != "-" ; end
+    def      authenticated_user?() self.class.authenticated_user?(self.remote_user) ; end
+
+    #
+    # "GET /xd_receiver.html HTTP/1.1"
+    # "GET /crossdomain.xml HTTP/1.1"
+    # "GET /favicon.ico HTTP/1.1"
+    # "GET /robots.txt HTTP/1.0"
+    #
+
+    AUTOMATED_REQUESTS = [
+      Regexp.compile('^[A-Z]+ \/xd_receiver.html'),
+      Regexp.compile('^[A-Z]+ \/crossdomain.xml'),
+      Regexp.compile('^[A-Z]+ \/favicon.ico'),
+      Regexp.compile('^[A-Z]+ \/robots.txt'),
+      nil
+    ].compact!
+  
+    def self.automated_request?(request) !AUTOMATED_REQUESTS.detect { |automated_request_regexp| request.match(automated_request_regexp) }.nil? end
+    def      automated_request?()        self.class.automated_request?(self.request) ; end
+
+    # 
+    # subdirectories of the "public" folder in the web root,
+    # which - in a typical Rails setup - are served by nginx
+    #
+
+    STATIC_REPOS = %w{
+      flash
+      html
+      images
+      javascripts
+      movies
+      newsletters
+      pictures
+      stylesheets
+      xml
+    }
+
+    STATIC_URIS = STATIC_REPOS.map { |repo| Regexp.compile("^\/#{repo}\/") }
+  
+    def self.static_uri?(uri) !STATIC_URIS.detect { |static_uri_regexp| uri.match(static_uri_regexp) }.nil? end
+    def      static_uri?()    self.class.static_uri?(self.uri); end
+
+    STATIC_REQUESTS = STATIC_REPOS.map { |repo| Regexp.compile("^[A-Z]+ \/#{repo}\/") }
+  
+    def self.static_request?(request) !STATIC_REQUESTS.detect { |static_request_regexp| request.match(static_request_regexp) }.nil? end
+    def      static_request?()        self.class.static_request?(self.request) ; end
+
+    NGINX_MAGIC_STATUS = '499'   # ex-standard HTTP response code specific to nginx, in addition to http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html
+    UNPROCESSABLE_ENTITY = '422' # not supported by 'net/http' (Net::HTTPResponse::CODE_TO_OBJ["422"] == nil), see also: http://www.ruby-forum.com/topic/98002 
+
+    # Informational 1xx
+    def self.information_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPInformation ; end
+    def      information_status?()       self.class.information_status?(self.status) ; end
+
+    # Successful 2xx
+    def self.success_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPSuccess ; end
+    def      success_status?()       self.class.success_status?(self.status) ; end
+
+    # Redirection 3xx
+    def self.redirect_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPRedirection ; end
+    def      redirect_status?()       self.class.redirect_status?(self.status) ; end
+
+    # Client Error 4xx
+    def self.client_error_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPClientError ; end
+    def      client_error_status?()       self.class.client_error_status?(self.status) ; end
+
+    # Internal Server Error 5xx
+    def self.server_error_status?(status) (status.to_s != NGINX_MAGIC_STATUS) and Net::HTTPResponse::CODE_TO_OBJ[status.to_s] <= Net::HTTPServerError ; end
+    def      server_error_status?()       self.class.server_error_status?(self.status) ; end
+  
+  end # class LogLine
+end # module NginxTail

data/ntail.gemspec

@@ -0,0 +1,75 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{ntail}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Peter Vandenberk"]
+  s.date = %q{2010-12-20}
+  s.default_executable = %q{ntail}
+  s.description = %q{A parsing tail(1)-like utility for nginx log files}
+  s.email = %q{pvandenberk@mac.com}
+  s.executables = ["ntail"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/ntail",
+    "lib/ntail.rb",
+    "lib/ntail/application.rb",
+    "lib/ntail/log_line.rb",
+    "ntail.gemspec",
+    "test/helper.rb",
+    "test/test_ntail.rb"
+  ]
+  s.homepage = %q{http://github.com/pvdb/ntail}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{A parsing tail(1)-like utility for nginx log files}
+  s.test_files = [
+    "test/helper.rb",
+    "test/test_ntail.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<rainbow>, [">= 0"])
+      s.add_development_dependency(%q<user-agent>, [">= 0"])
+    else
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<rainbow>, [">= 0"])
+      s.add_dependency(%q<user-agent>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<rainbow>, [">= 0"])
+    s.add_dependency(%q<user-agent>, [">= 0"])
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'ntail'
+
+class Test::Unit::TestCase
+end

data/test/test_ntail.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestNtail < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification 
+name: ntail
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Peter Vandenberk
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-20 00:00:00 +00:00
+default_executable: ntail
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  name: shoulda
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id001
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  name: bundler
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  requirement: *id002
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  name: jeweler
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 5
+        - 1
+        version: 1.5.1
+  requirement: *id003
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  name: rcov
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id004
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  name: rainbow
+  version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id005
+- !ruby/object:Gem::Dependency 
+  type: :development
+  prerelease: false
+  name: user-agent
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id006
+description: A parsing tail(1)-like utility for nginx log files
+email: pvandenberk@mac.com
+executables: 
+- ntail
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- .document
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- bin/ntail
+- lib/ntail.rb
+- lib/ntail/application.rb
+- lib/ntail/log_line.rb
+- ntail.gemspec
+- test/helper.rb
+- test/test_ntail.rb
+has_rdoc: true
+homepage: http://github.com/pvdb/ntail
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A parsing tail(1)-like utility for nginx log files
+test_files: 
+- test/helper.rb
+- test/test_ntail.rb