ntail 0.0.11 → 0.0.12

data/Gemfile

@@ -5,13 +5,18 @@ source "http://rubygems.org"
 gem "rainbow", ">= 0"
 gem "user-agent", ">= 0"
 gem "treetop", "~> 1.4.9"
+gem 'sequel'
+gem 'mongoid'
+gem 'sqlite3-ruby', :require => 'sqlite3'
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
+  gem "rake", ">= 0.9.2"
   gem "shoulda", ">= 0"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.5.1"
   gem "rcov", ">= 0"
   gem "geoip", ">= 0"
+  gem "rspec", ">= 2.5"
 end

data/Gemfile.lock

@@ -1,19 +1,47 @@
 GEM
   remote: http://rubygems.org/
   specs:
-    geoip (0.8.7)
+    activemodel (3.0.9)
+      activesupport (= 3.0.9)
+      builder (~> 2.1.2)
+      i18n (~> 0.5.0)
+    activesupport (3.0.9)
+    bson (1.3.1)
+    builder (2.1.2)
+    diff-lcs (1.1.2)
+    geoip (1.0.0)
     git (1.2.5)
-    jeweler (1.5.1)
+    i18n (0.5.0)
+    jeweler (1.5.2)
       bundler (~> 1.0.0)
       git (>= 1.2.5)
       rake
+    mongo (1.3.1)
+      bson (>= 1.3.1)
+    mongoid (2.1.2)
+      activemodel (~> 3.0)
+      mongo (~> 1.3)
+      tzinfo (~> 0.3.22)
     polyglot (0.3.1)
-    rainbow (1.1)
-    rake (0.8.7)
+    rainbow (1.1.1)
+    rake (0.9.2)
     rcov (0.9.9)
+    rspec (2.5.0)
+      rspec-core (~> 2.5.0)
+      rspec-expectations (~> 2.5.0)
+      rspec-mocks (~> 2.5.0)
+    rspec-core (2.5.1)
+    rspec-expectations (2.5.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.5.0)
+    sequel (3.25.0)
     shoulda (2.11.3)
+    sqlite3 (1.3.4)
+    sqlite3-ruby (1.3.3)
+      sqlite3 (>= 1.3.3)
     treetop (1.4.9)
       polyglot (>= 0.3.1)
+    tzinfo (0.3.29)
     user-agent (1.0.0)
 
 PLATFORMS
@@ -23,8 +51,13 @@ DEPENDENCIES
   bundler (~> 1.0.0)
   geoip
   jeweler (~> 1.5.1)
+  mongoid
   rainbow
+  rake (>= 0.9.2)
   rcov
+  rspec (>= 2.5)
+  sequel
   shoulda
+  sqlite3-ruby
   treetop (~> 1.4.9)
   user-agent

data/README.md

@@ -0,0 +1,158 @@
+ntail
+=====
+
+A `tail(1)`-like utility for nginx log files that supports parsing, filtering and formatting of individual 
+log lines (in nginx's so-called ["combined" log format](http://wiki.nginx.org/NginxHttpLogModule#log_format)).
+
+<a name="intro"/>
+
+Check it out, yo!
+-----------------
+
+Instead of this...
+
+<pre style="background-color: black; color: white; padding: 15px; width: 1100px; overflow: hidden; text-overflow: ellipsis;">
+<span style="color:white;">$ tail -f /var/log/nginx/access.log</span>
+<span style="color: green;">192.0.32.10 - - [21/Jan/2011:14:07:34 +0000] "GET / HTTP/1.1" 200 3700 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10" "-"</span>
+<span style="color: green;">192.0.32.10 - - [21/Jan/2011:14:07:34 +0000] "GET /nginx-logo.png HTTP/1.1" 200 370 "http://localhost/" "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10" "-"</span>
+<span style="color: green;">192.0.32.10 - - [21/Jan/2011:14:07:34 +0000] "GET /poweredby.png HTTP/1.1" 200 3034 "http://localhost/" "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10" "-"</span>
+<span style="color: green;">192.0.32.10 - - [21/Jan/2011:14:07:34 +0000] "GET /favicon.ico HTTP/1.1" 404 3650 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10" "-"</span>
+<span style="color: green;">192.0.32.10 - - [21/Jan/2011:14:19:04 +0000] "GET /nginx-logo.png HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.237 Safari/534.10" "-"</span>
+<span style="color:white;">$ _</span>
+</pre>
+
+... you get this:
+
+<pre style="background-color: black; padding: 15px; width: 800px;">
+<span style="color:white;">$ tail -f /var/log/nginx/access.log <strong>| ntail</strong></span>
+<span style="color: green;">2011-01-21 14:07:34 -       192.0.32.10 - 200 - GET / - (Chrome, Linux) - -</span>
+<span style="color: green;">2011-01-21 14:07:34 -       192.0.32.10 - 200 - GET /nginx-logo.png - (Chrome, Linux) - localhost</span>
+<span style="color: green;">2011-01-21 14:07:34 -       192.0.32.10 - 200 - GET /spanoweredby.png - (Chrome, Linux) - localhost</span>
+<span style="color: red;">2011-01-21 14:07:34 -       192.0.32.10 - 404 - GET /favicon.ico - (Chrome, Linux) - -</span>
+<span style="color: orange;">2011-01-21 14:19:04 -       192.0.32.10 - 304 - GET /nginx-logo.png - (Chrome, Linux) - -</span>
+<span style="color:white;">$ _</span>
+</pre>
+
+<a name="installation"/>
+
+Installation
+------------
+
+Installing the gem also installs the `ntail` executable, typically as `/usr/bin/ntail` or `/usr/local/bin/ntail`:
+
+    $ gem install ntail
+
+To ensure easy execution of the `ntail` script, add the actual installation directory to your shell's `$PATH` variable.
+
+<a name="basic"/>
+
+Basic Usage
+-----------
+
+* process an entire nginx log file and print each parsed and formatted line to STDOUT
+
+        $ ntail /var/log/nginx/access.log
+
+* tail an "active" nginx log file and print each new line to STDOUT _(stop with ^C)_
+
+        $ tail -f /var/log/nginx/access.log | ntail
+
+<a name="advanced"/>
+
+Advanced Examples
+-----------------
+
+* read from STDIN and print each line to STDOUT _(stop with ^D)_
+
+        $ ntail
+
+* read from STDIN and print out the length of each line _(to illustrate -e option)_
+
+        $ ntail -e 'puts size'
+
+* read from STDIN but only print out non-empty lines _(to illustrate -f option)_
+
+        $ ntail -f 'size != 0'
+
+* the following invocations behave exactly the same _(to illustrate -e and -f options)_
+
+        $ ntail
+        $ ntail -f 'true' -e 'puts self'
+
+* print out all HTTP requests that are coming from a given IP address
+
+        $ ntail -f 'remote_address == "208.67.222.222"' /var/log/nginx/access.log
+
+* find all HTTP requests that resulted in a '5xx' HTTP error/status code _(e.g. Rails 500 errors)_
+
+        $ gunzip -S .gz -c access.log-20101216.gz | ntail -f 'server_error_status?'
+
+* generate a summary report of HTTP status codes, for all non-200 HTTP requests
+
+        $ ntail -f 'status != "200"' -e 'puts status' access.log | sort | uniq -c
+        76 301
+        16 302
+         2 304
+         1 406
+
+* print out GeoIP country and city information for each HTTP request _(depends on the optional `geoip` gem)_
+
+        $ ntail -e 'puts [to_country_s, to_city_s].join("\t")' /var/log/nginx/access.log
+        United States   Los Angeles
+        United States   Houston
+        Germany         Berlin
+        United Kingdom  London
+
+* print out the IP address and the corresponding host name for each HTTP request _(slows things down considerably, due to `nslookup` call)_
+
+        $ ntail -e 'puts [remote_address, to_host_s].join("\t")' /var/log/nginx/access.log
+        66.249.72.196   crawl-66-249-72-196.googlebot.com
+        67.192.120.134  s402.pingdom.com
+        75.31.109.144   adsl-75-31-109-144.dsl.irvnca.sbcglobal.net
+
+* parse an access log file, and pipe its raw output (indirectly - via the `parsed.log` file) into the `gltail` realtime logfile visualizer
+
+        $ ntail -v --raw --sleep 0.1 /var/log/nginx/access.log > parsed.log
+
+<a name="todo"/>
+
+TODO
+----
+    
+* implement a native `"-f"` option for ntail, similar to that of `tail(1)`, using e.g. flori's [file-tail gem](https://github.com/flori/file-tail)
+* implement a `"-i"` option ("ignore exceptions"/"continue processing"), if handling a single line raises an exception
+* or indeed a reverse `"-r"` option ("re-raise exception"), to immediately stop processing and raising the exception for investigation
+* implement (better) support for custom nginx log formats, in addition to [nginx's default "combined" log format](http://wiki.nginx.org/NginxHttpLogModule#log_format).
+
+<a name="acknowledgements"/>
+
+Acknowledgements
+----------------
+
+* ntail's parsing feature is inspired by an nginx log parser written by [Richard Taylor (moomerman)](https://github.com/moomerman)
+* parsing and expanding ntail's formatting string is done using nathansobo's quite brilliant [treetop gem](https://github.com/nathansobo/treetop)
+* ntail's raw line output is compatible with Fudge's fun and useful [gltail gem](https://github.com/Fudge/gltail)
+* Kudos to [Ed James (edjames)](https://github.com/edjames) for recommending the use of [instance_eval][eval] to clean up the DSL
+
+[eval]: https://github.com/pvdb/ntail/commit/b0f40522012b9858c433808cd1f5c21cb455fadd "use instance_eval to simplify the DSL"
+
+<a name="contributing"/>
+
+Contributing to ntail
+---------------------
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+<a name="copyright"/>
+
+Copyright
+---------
+
+Copyright (c) 2011 Peter Vandenberk. See LICENSE.txt for further details.
+

data/Rakefile

@@ -26,6 +26,16 @@ Jeweler::Tasks.new do |gem|
 end
 Jeweler::RubygemsDotOrgTasks.new
 
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "ntail #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
@@ -33,21 +43,24 @@ Rake::TestTask.new(:test) do |test|
   test.verbose = true
 end
 
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |test|
+  test.verbose = true
+end
+
 require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
+Rcov::RcovTask.new(:test_rcov) do |test|
   test.libs << 'test'
   test.pattern = 'test/**/test_*.rb'
+  test.rcov_opts = %w{--exclude test\/,spec\/ -T}
+  test.verbose = true
+end
+Rcov::RcovTask.new(:spec_rcov) do |test|
+  test.libs << 'spec'
+  test.pattern = 'spec/**/*_spec.rb'
+  test.rcov_opts = %w{--exclude test\/,spec\/ -T}
   test.verbose = true
 end
 
-task :default => :test
-
-require 'rake/rdoctask'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+task :default => [:test, :spec]
 
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "ntail #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end

data/VERSION

@@ -1 +1 @@
-0.0.11
+0.0.12

data/bin/ntail

@@ -9,6 +9,6 @@ rescue LoadError
   require 'ntail'
 end
 
-exit NginxTail::Application.run!
+exit NginxTail::Application.new(ARGV).run!
 
 # That's all, Folks!

data/lib/ntail/application.rb

@@ -3,145 +3,122 @@ require 'optparse'
 
 module NginxTail
   class Application
-    
-    def self.options
-      # application options from the command line
-      @@options ||= OpenStruct.new
-    end
-    
-    def self.ntail_options
-      # shamelessly copied from lib/rake.rb (rake gem)
-      [
-        ['--verbose', '-v', "Run verbosely (log messages to STDERR).",
-          lambda { |value|
-            self.options.verbose = true
-          }
-        ],
-        ['--dry-run', '-n', "Dry-run: process files, but don't actually parse the lines",
-          lambda { |value|
-            self.options.dry_run = true
-          }
-        ],
-        ['--parse-only', '-p', "Parse only: parse all lines, but don't actually process them",
-          lambda { |value|
-            self.options.parse_only = true
-          }
-        ],
-        ['--version', '-V', "Display the program version.",
-          lambda { |value|
-            puts "#{NTAIL_NAME}, version #{NTAIL_VERSION}"
-            self.options.running = false
-          }
-        ],
-        ['--line-number', '-l LINE_NUMBER', "Only process the line with the given line number",
-          lambda { |value|
-            self.options.line_number = value.to_i
-          }
-        ],
-        ['--filter',  '-f CODE', "Ruby code block for filtering (parsed) lines - needs to return true or false.",
-          lambda { |value|
-            self.options.filter = eval "Proc.new #{value}"
-          }
-        ],
-        ['--execute',  '-e CODE', "Ruby code block for processing each (parsed) line.",
-          lambda { |value|
-            self.options.code = eval "Proc.new #{value}"
-          }
-        ],
-      ]
+
+    include NginxTail::Options
+
+    # default application options...
+    DEFAULT_OPTIONS = {
+      :interrupted => false,
+      :running => true,
+      :nginx => true,
+      :exit => 0,
+    }
+
+    # parsed application options...
+    @options = nil
+
+    def respond_to?(symbol, include_private = false)
+      @options.respond_to?(symbol) || super
     end
 
-    def self.parse_options
-      
-      # application defaults...
-      self.options.interrupted = false
-      self.options.running = true
-      self.options.exit = 0
-      
-      OptionParser.new do |opts|
-        opts.banner = "ntail {options} {file(s)} ..."
-        opts.separator ""
-        opts.separator "Options are ..."
-
-        opts.on_tail("-h", "--help", "-H", "Display this help message.") do
-          puts opts
-          self.options.running = false
-        end
+    def method_missing(methodId)
+      respond_to?(methodId) ? @options.send(methodId.to_sym) : super
+    end
 
-        self.ntail_options.each { |args| opts.on(*args) }
-      end.parse!
+    def initialize(argv = [])
+      @options = parse_options(argv, DEFAULT_OPTIONS)
     end
-    
-    def self.run!
-      
-      self.parse_options
-      
+
+    def run!
+
+      LogLine.set_log_pattern(@options.nginx)
+
       ['TERM', 'INT'].each do |signal|
         Signal.trap(signal) do
-          self.options.running = false ; self.options.interrupted = true
+          @options.running = false ; @options.interrupted = true
           $stdin.close if ARGF.file == $stdin # ie. reading from STDIN
         end
       end
-      
+
       files_read = lines_read = lines_processed = lines_ignored = parsable_lines = unparsable_lines = 0
-      
-      while self.options.running and ARGF.gets
+
+      current_filename = nil ; current_line_number = 0 ; file_count = ARGV.count
+
+      while @options.running and ARGF.gets
         if ARGF.file.lineno == 1
+          current_filename = ARGF.filename ; current_line_number = 0
           files_read += 1
-          if self.options.verbose
+          if @options.verbose
             $stderr.puts "[INFO] now processing file #{ARGF.filename}"
           end
         end
-        raw_line = $_.chomp ; lines_read += 1
-        unless self.options.dry_run
-          if !self.options.line_number or self.options.line_number == ARGF.lineno
+        raw_line = $_.chomp ; lines_read += 1 ; current_line_number += 1
+        unless @options.dry_run
+          if !@options.line_number or @options.line_number == ARGF.lineno
             begin
-              log_line = NginxTail::LogLine.new(raw_line)
+              log_line = NginxTail::LogLine.new(raw_line, current_filename, current_line_number)
               if log_line.parsable
                 parsable_lines += 1
-                unless self.options.parse_only
-                  if !self.options.filter || self.options.filter.call(log_line)
+                unless @options.parse_only
+                  if !@options.filter || log_line.instance_eval(@options.filter)
                     lines_processed += 1
-                    if self.options.code
-                      self.options.code.call(log_line)
+                    if @options.code
+                      log_line.instance_eval(@options.code)
+                    elsif @options.raw
+                      $stdout.puts raw_line
+                      sleep @options.sleep if @options.sleep
                     else
-                      puts log_line
+                      puts log_line.to_s(:color => true)
                     end
                   else
                     lines_ignored += 1
-                    if self.options.verbose
+                    if @options.verbose
                       $stderr.puts "[WARNING] ignoring line ##{lines_read}"
                     end
                   end
                 end
               else
                 unparsable_lines += 1
-                if self.options.verbose
+                if @options.verbose
                   $stderr.puts "[ERROR] cannot parse '#{raw_line}'"
                 end
               end
             rescue
               $stderr.puts "[ERROR] processing line #{ARGF.file.lineno} of file #{ARGF.filename} resulted in #{$!.message}"
               $stderr.puts "[ERROR] " + raw_line
-              self.options.exit = -1
-              self.options.running = false
+              @options.exit = -1
+              @options.running = false
               raise $! # TODO if the "re-raise exceptions" option has been set...
             end
           end
         end
+        if @options.progress
+          progress_line = [
+            " Processing file ".inverse + (" %d/%d" % [files_read, file_count]),
+            " Current filename ".inverse + " " + current_filename.to_s,
+            " Line number ".inverse + " " + current_line_number.to_s,
+            " Lines processed ".inverse + " " + lines_read.to_s
+          ].join(" \342\200\242 ")
+          max_length = [max_length || 0, progress_line.size].max
+          $stderr.print progress_line
+          $stderr.print " " * (max_length - progress_line.size)
+          $stderr.print "\r"
+        end
       end
-      
-      if self.options.verbose
-        $stderr.puts if self.options.interrupted
-        $stderr.print "[INFO] read #{lines_read} lines in #{files_read} files"
-        $stderr.print " (interrupted)" if self.options.interrupted ; $stderr.puts
+
+      $stderr.puts if @options.progress
+
+      if @options.verbose
+        $stderr.puts if @options.interrupted
+        $stderr.print "[INFO] read #{lines_read} line(s) in #{files_read} file(s)"
+        $stderr.print " (interrupted)" if @options.interrupted ; $stderr.puts
         $stderr.puts "[INFO] #{parsable_lines} parsable lines, #{unparsable_lines} unparsable lines"
         $stderr.puts "[INFO] processed #{lines_processed} lines, ignored #{lines_ignored} lines"
       end
-      
-      return self.options.exit
-      
-    end # def run
-    
+
+      return @options.exit
+
+    end
+
   end
 end

data/lib/ntail/body_bytes_sent.rb

@@ -5,10 +5,10 @@ module NginxTail
       base.class_eval do
 
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'body_bytes_sent'" unless base.instance_methods.include? 'body_bytes_sent'
+        raise "Class #{base.name} should implement instance method 'body_bytes_sent'" unless base.instance_methods.map(&:to_s).include? 'body_bytes_sent'
         
       end
     end
         
   end
-end
+end