ntail 0.0.11 → 0.0.12

data/lib/ntail/log_line.rb

@@ -6,34 +6,47 @@ require 'rainbow'
 module NginxTail
   class LogLine
 
-    def self.component_to_module_name(component)
-      # this mimicks the ActiveSupport::Inflector.camelize() method in Rails...
-      component.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
-    end
-    
-    def self.component_to_ntail_module(component)
-      # this mimicks the ActiveSupport::Inflector.constantize() method in Rails...
-      NginxTail.const_get(self.component_to_module_name(component))
-    end
-
     attr_reader :raw_line
     attr_reader :parsable
+    attr_reader :filename
+    attr_reader :line_number
+
+    COMPONENTS = [ # formatting token:
+
+      # NGINX
+
+      :remote_addr,     # %a
+      :remote_user,     # %u
+      :time_local,      # %t
+      :request,         # %r
+      :status,          # %s
+      :body_bytes_sent, # %b
+      :http_referer,    # %R
+      :http_user_agent, # %U
+      :proxy_addresses, # %p
+
+      # APACHE
+
+      :server_name,     # %V
+      # :remote_addr,     # %h
+      :remote_log_name, # %l
+      # :remote_user,     # %u
+      # :time_local,      # %t
+      # :request,         # %r
+      # :status,          # %s
+      # :body_bytes_sent, # %b
+      # :http_referer,    # %%{Referer}i
+      # :http_user_agent, # %%{User-agent}i
+      :http_cookie,     # %{Cookie}i
+      :time_taken,      # %D
 
-    COMPONENTS = [
-      :remote_addr,
-      :remote_user,
-      :time_local,
-      :request,
-      :status,
-      :body_bytes_sent,
-      :http_referer,
-      :http_user_agent,
-      :proxy_addresses,
     ]
-    
+
     COMPONENTS.each do |symbol|
       attr_reader symbol
-      include component_to_ntail_module(symbol)
+      if (module_for_symbol = Inflections.component_to_ntail_module(symbol))
+        include module_for_symbol
+      end
     end
 
     include KnownIpAddresses # module to identify known IP addresses
@@ -44,23 +57,47 @@ module NginxTail
       :uri,
       :http_version,
     ]
-    
+
     SUBCOMPONENTS.each do |symbol|
       attr_reader symbol
-      include component_to_ntail_module(symbol)
+      include Inflections.component_to_ntail_module(symbol)
     end
 
+    #
+    # http://httpd.apache.org/docs/2.0/mod/mod_log_config.html - we currently only support the following custom log format...
+    #
+    # "%V %h %l %u %t \"%r\" %s %b \"%{Referer}i\" \"%{User-agent}i\" \"%{Cookie}i\" %I %O %D %{deflate_ratio}n%%"
+    #
+
+    APACHE_LOG_PATTERN = Regexp.compile(/\A([\S]*) ([\S]+) ([\S]+|-) ([\S]+|-) \[([^\]]+)\] "(.*)" ([\d]+) ([\d]+|-) "(.*?)" "(.*?)" "(.*?)" [\d]+ [\d]+ ([\d]+) .*\Z/)
+
     #
     # http://wiki.nginx.org/NginxHttpLogModule#log_format - we currently only support the default "combined" log format...
     #
 
     NGINX_LOG_PATTERN = Regexp.compile(/\A(\S+) - (\S+) \[([^\]]+)\] "([^"]+)" (\S+) (\S+) "([^"]*?)" "([^"]*?)"( "([^"]*?)")?\Z/)
+
+    #
+    # the actual pattern used for line matching, either nginx (default) or apache
+    #
+
+    @@log_pattern = NGINX_LOG_PATTERN
+
+    def self.set_log_pattern(nginx_format)
+      @@log_pattern = nginx_format ? NGINX_LOG_PATTERN : APACHE_LOG_PATTERN
+    end
+
     NGINX_REQUEST_PATTERN = Regexp.compile(/\A(\S+) (.*?) (\S+)\Z/)
     NGINX_PROXY_PATTERN = Regexp.compile(/\A "([^"]*)"\Z/)
 
-    def initialize(line)
-      @parsable = if NGINX_LOG_PATTERN.match(@raw_line = line)
-        @remote_addr, @remote_user, @time_local, @request, @status, @body_bytes_sent, @http_referer, @http_user_agent, @proxy_addresses = $~.captures
+    def initialize(line, filename = nil, line_number = nil)
+      @filename = filename ; @line_number = line_number
+      @parsable = if @@log_pattern.match(@raw_line = line)
+        if @@log_pattern == NGINX_LOG_PATTERN
+          @remote_addr, @remote_user, @time_local, @request, @status, @body_bytes_sent, @http_referer, @http_user_agent, @proxy_addresses = $~.captures
+        elsif @@log_pattern == APACHE_LOG_PATTERN
+          @server_name, @remote_addr, @remote_log_name, @remote_user, @time_local, @request, @status, @body_bytes_sent, @http_referer, @http_user_agent, @http_cookie, @time_taken = $~.captures
+        end
         if NGINX_REQUEST_PATTERN.match(@request)
           # counter example (ie. HTTP request that cannot by parsed)
           # 91.203.96.51 - - [21/Dec/2010:05:26:53 +0000] "-" 400 0 "-" "-"
@@ -74,35 +111,46 @@ module NginxTail
         false
       end
     end
-    
+
     alias_method :remote_address, :remote_addr # a non-abbreviated alias, for convenience and readability...
-    
+
     # for now, until we make it fancier...
     def method_missing(method, *params)
       raw_line.send method, *params
     end
-    
+
     @@parser = FormattingParser.new
-    
-    def to_s()
+
+    @@result = @@format = nil
+
+    def self.format= format
+      unless @@result = @@parser.parse(@@format = format)
+        raise @@parser.terminal_failures.join("\n")
+      else
+        def @@result.value(log_line, color)
+          elements.map { |element| element.value(log_line, color) }.join
+        end
+      end
+    end
+
+    self.format = "%t - %a - %s - %r - %U - %R"
+
+    def to_s(options = {})
+
       # simple but boring:
       # raw_line.to_s
-      color = if redirect_status?
+
+      # a bit less boring:
+      color = options[:color] && if redirect_status?
         :yellow
       elsif !success_status?
         :red
       else
         :default
       end
-      unless result = @@parser.parse("%d - %a - %s - %r - %u - %f")
-        raise @@parser.terminal_failures.join("\n")
-      else
-        def result.value(log_line, color)
-          elements.map { |element| element.value(log_line, color) }.join
-        end
-      end
-      result.value(self, color) 
+      @@result.value(self, color)
+
     end
-  
+
   end # class LogLine
 end # module NginxTail

data/lib/ntail/node.rb

@@ -5,5 +5,8 @@ module Formatting
     def value(log_line, color)
       raise "SubclassResponsibility" # override in node "subclasses"...
     end
+    def foreground(string, color)
+      color ? string.foreground(color) : string
+    end
   end
 end

data/lib/ntail/options.rb

@@ -0,0 +1,82 @@
+module NginxTail
+  module Options
+
+    def parse_options(argv, defaults = {})
+
+      options = OpenStruct.new(defaults)
+
+      OptionParser.new do |opts|
+
+        opts.banner = "Usage: ntail {options} {file(s)}"
+        opts.separator ""
+        opts.separator "Options are ..."
+
+        opts.on '--verbose', '-v', "Run verbosely (log messages to STDERR)." do |value|
+          options.verbose = true
+        end
+
+        opts.on '--filter',  '-f CODE', "Ruby code block for filtering (parsed) lines - needs to return true or false." do |value|
+          options.filter = value
+        end
+
+        opts.on '--execute',  '-e CODE', "Ruby code block for processing each (parsed) line." do |value|
+          options.code = value
+        end
+
+        opts.on '--line-number', '-l LINE_NUMBER', "Only process the line with the given line number" do |value|
+          options.line_number = value.to_i
+        end
+
+        opts.on '--dry-run', '-n', "Dry-run: process files, but don't actually parse the lines" do |value|
+          options.dry_run = true
+        end
+
+        opts.on '--parse-only', '-p', "Parse only: parse all lines, but don't actually process them" do |value|
+          options.parse_only = true
+        end
+
+        opts.on '--raw', '-r', "Parse lines, and - for parseable ones - print out the raw input" do |value|
+          options.raw = true
+        end
+
+        opts.on '--apache', "Try to match lines using the Apache log format instead of nginx (the default)" do |value|
+          options.nginx = false
+        end
+
+        opts.on '--sleep [SECONDS]', '-s', Float, "Sleeps for the given number of seconds before processing the next line (--raw only)" do |value|
+          options.sleep = value
+        end
+
+        opts.on '--progress', '-p', String, "In-flight progress animation during parsing" do |value|
+          unless $stdout.tty?
+            Sickill::Rainbow.enabled = true
+            options.progress = true
+          end
+        end
+
+        opts.on '--static-repo [REPO]', String, "Add [REPO] to the list of static repos" do |value|
+          NginxTail::LogLine.add_static_repo(value)
+        end
+
+        opts.on '--persist', '-P', String, "Persist the parsed lines for future use" do |value|
+          options.persist = true
+        end
+
+        opts.on '--version', '-V', "Display the program version." do |value|
+          puts "#{NTAIL_NAME}, version #{NTAIL_VERSION}"
+          options.running = false
+        end
+
+        opts.on_tail("-h", "--help", "-H", "Display this help message.") do
+          puts opts
+          options.running = false
+        end
+
+      end.parse!(argv)
+
+      return options
+
+    end
+
+  end # module Options
+end # module NginxTail

data/lib/ntail/proxy_addresses.rb

@@ -5,10 +5,10 @@ module NginxTail
       base.class_eval do
 
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'proxy_addresses'" unless base.instance_methods.include? 'proxy_addresses'
+        raise "Class #{base.name} should implement instance method 'proxy_addresses'" unless base.instance_methods.map(&:to_s).include? 'proxy_addresses'
         
       end
     end
         
   end
-end
+end

data/lib/ntail/remote_addr.rb

@@ -35,7 +35,7 @@ module NginxTail
         end
         
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'remote_addr'" unless base.instance_methods.include? 'remote_addr'
+        raise "Class #{base.name} should implement instance method 'remote_addr'" unless base.instance_methods.map(&:to_s).include? 'remote_addr'
 
       end
     end
@@ -53,4 +53,4 @@ module NginxTail
     end
     
   end
-end
+end

data/lib/ntail/remote_user.rb

@@ -42,7 +42,7 @@ module NginxTail
         end
 
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'remote_user'" unless base.instance_methods.include? 'remote_user'
+        raise "Class #{base.name} should implement instance method 'remote_user'" unless base.instance_methods.map(&:to_s).include? 'remote_user'
 
       end
     end
@@ -60,4 +60,4 @@ module NginxTail
     end
 
   end
-end
+end

data/lib/ntail/request.rb

@@ -11,7 +11,7 @@ module NginxTail
         end
 
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'request'" unless base.instance_methods.include? 'request'
+        raise "Class #{base.name} should implement instance method 'request'" unless base.instance_methods.map(&:to_s).include? 'request'
         
       end
     end
@@ -30,4 +30,4 @@ module NginxTail
     end
         
   end
-end
+end

data/lib/ntail/status.rb

@@ -37,7 +37,7 @@ module NginxTail
         end
 
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'status'" unless base.instance_methods.include? 'status'
+        raise "Class #{base.name} should implement instance method 'status'" unless base.instance_methods.map(&:to_s).include? 'status'
         
       end
     end

data/lib/ntail/time_local.rb

@@ -21,7 +21,7 @@ module NginxTail
         end
 
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'time_local'" unless base.instance_methods.include? 'time_local'
+        raise "Class #{base.name} should implement instance method 'time_local'" unless base.instance_methods.map(&:to_s).include? 'time_local'
         
       end
     end
@@ -35,4 +35,4 @@ module NginxTail
     end
     
   end
-end
+end

data/lib/ntail/uri.rb

@@ -64,15 +64,22 @@ module NginxTail
           html
           images
           javascripts
+          js
           movies
           newsletters
           pictures
           stylesheets
+          css
           xml
         }
 
         @@static_uris = @@static_repos.map { |repo| Regexp.compile("^\/#{repo}\/") }
 
+        def self.add_static_repo(repo)
+          # TODO make this DRY...
+          @@static_uris << Regexp.compile("^\/#{repo}\/")
+        end
+
         def self.static_uri?(uri)
           !@@static_uris.detect { |static_uri_regexp| uri.match(static_uri_regexp) }.nil?
         end
@@ -82,7 +89,7 @@ module NginxTail
         end
 
         # this ensures the below module methods actually make sense...
-        raise "Class #{base.name} should implement instance method 'uri'" unless base.instance_methods.include? 'uri'
+        raise "Class #{base.name} should implement instance method 'uri'" unless base.instance_methods.map(&:to_s).include? 'uri'
         
       end
     end

data/lib/ntail.rb

@@ -1,6 +1,20 @@
 NTAIL_NAME = 'ntail'
 NTAIL_VERSION = '0.0.1'
 
+# module methods to be used as functions...
+module NginxTail
+  module Inflections
+    def self.component_to_module_name(component)
+      # this mimicks the ActiveSupport::Inflector.camelize() method in Rails...
+      component.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    end
+    def self.component_to_ntail_module(component)
+      # this mimicks the ActiveSupport::Inflector.constantize() method in Rails...
+      NginxTail.const_get(self.component_to_module_name(component)) rescue nil
+    end
+  end
+end
+
 # so-called components...
 require 'ntail/remote_addr'
 require 'ntail/remote_user'
@@ -26,7 +40,8 @@ require 'ntail/node.rb'
 require 'ntail/formatting.rb'
 
 # the core classes...
-require 'ntail/log_line'
-require 'ntail/application'
+require 'ntail/log_line.rb'
+require 'ntail/options.rb'
+require 'ntail/application.rb'
 
-# That's all, Folks!
+# That's all, Folks!

data/ntail.gemspec

@@ -5,25 +5,25 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ntail}
-  s.version = "0.0.11"
+  s.version = "0.0.12"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Peter Vandenberk"]
-  s.date = %q{2011-01-19}
+  s.date = %q{2011-08-08}
   s.default_executable = %q{ntail}
   s.description = %q{A tail(1)-like utility for nginx log files. It supports parsing, filtering and formatting individual log lines.}
   s.email = %q{pvandenberk@mac.com}
   s.executables = ["ntail"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
-    "README.rdoc"
+    "README.md"
   ]
   s.files = [
     ".document",
     "Gemfile",
     "Gemfile.lock",
     "LICENSE.txt",
-    "README.rdoc",
+    "README.md",
     "Rakefile",
     "VERSION",
     "bin/ntail",
@@ -40,6 +40,7 @@ Gem::Specification.new do |s|
     "lib/ntail/local_ip_addresses.rb",
     "lib/ntail/log_line.rb",
     "lib/ntail/node.rb",
+    "lib/ntail/options.rb",
     "lib/ntail/proxy_addresses.rb",
     "lib/ntail/remote_addr.rb",
     "lib/ntail/remote_user.rb",
@@ -48,10 +49,14 @@ Gem::Specification.new do |s|
     "lib/ntail/time_local.rb",
     "lib/ntail/uri.rb",
     "ntail.gemspec",
+    "spec/application_spec.rb",
+    "spec/spec_helper.rb",
     "test/helper.rb",
+    "test/ntail/test_formatting.rb",
     "test/ntail/test_http_method.rb",
     "test/ntail/test_http_referer.rb",
     "test/ntail/test_http_user_agent.rb",
+    "test/ntail/test_http_version.rb",
     "test/ntail/test_known_ip_addresses.rb",
     "test/ntail/test_local_ip_addresses.rb",
     "test/ntail/test_log_line.rb",
@@ -66,13 +71,17 @@ Gem::Specification.new do |s|
   s.homepage = %q{http://github.com/pvdb/ntail}
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
+  s.rubygems_version = %q{1.5.2}
   s.summary = %q{A tail(1)-like utility for nginx log files}
   s.test_files = [
+    "spec/application_spec.rb",
+    "spec/spec_helper.rb",
     "test/helper.rb",
+    "test/ntail/test_formatting.rb",
     "test/ntail/test_http_method.rb",
     "test/ntail/test_http_referer.rb",
     "test/ntail/test_http_user_agent.rb",
+    "test/ntail/test_http_version.rb",
     "test/ntail/test_known_ip_addresses.rb",
     "test/ntail/test_local_ip_addresses.rb",
     "test/ntail/test_log_line.rb",
@@ -86,37 +95,51 @@ Gem::Specification.new do |s|
   ]
 
   if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<rainbow>, [">= 0"])
       s.add_runtime_dependency(%q<user-agent>, [">= 0"])
       s.add_runtime_dependency(%q<treetop>, ["~> 1.4.9"])
+      s.add_runtime_dependency(%q<sequel>, [">= 0"])
+      s.add_runtime_dependency(%q<mongoid>, [">= 0"])
+      s.add_runtime_dependency(%q<sqlite3-ruby>, [">= 0"])
+      s.add_development_dependency(%q<rake>, [">= 0.9.2"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_development_dependency(%q<geoip>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 2.5"])
     else
       s.add_dependency(%q<rainbow>, [">= 0"])
       s.add_dependency(%q<user-agent>, [">= 0"])
       s.add_dependency(%q<treetop>, ["~> 1.4.9"])
+      s.add_dependency(%q<sequel>, [">= 0"])
+      s.add_dependency(%q<mongoid>, [">= 0"])
+      s.add_dependency(%q<sqlite3-ruby>, [">= 0"])
+      s.add_dependency(%q<rake>, [">= 0.9.2"])
       s.add_dependency(%q<shoulda>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
       s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<geoip>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 2.5"])
     end
   else
     s.add_dependency(%q<rainbow>, [">= 0"])
     s.add_dependency(%q<user-agent>, [">= 0"])
     s.add_dependency(%q<treetop>, ["~> 1.4.9"])
+    s.add_dependency(%q<sequel>, [">= 0"])
+    s.add_dependency(%q<mongoid>, [">= 0"])
+    s.add_dependency(%q<sqlite3-ruby>, [">= 0"])
+    s.add_dependency(%q<rake>, [">= 0.9.2"])
     s.add_dependency(%q<shoulda>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
     s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<geoip>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 2.5"])
   end
 end
 

data/spec/application_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe NginxTail::Application do
+
+  %w{ exit running interrupted }.each do |option|
+    it "should respond to #{option} option method" do
+      NginxTail::Application.new.should respond_to(option.to_sym)
+    end
+  end
+
+  it "has a default 'exit' value of 0" do
+    NginxTail::Application.new.exit.should eq(0)
+  end
+  
+  it "has a default 'running' value of true" do
+    NginxTail::Application.new.running.should eq(true)
+  end
+
+  it "has a default 'interrupted' value of false" do
+    NginxTail::Application.new.interrupted.should eq(false)
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'bundler'
+
+require 'rspec'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'ntail'

data/test/helper.rb

@@ -12,6 +12,7 @@ require 'shoulda'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
+
 require 'ntail'
 
 class Test::Unit::TestCase
@@ -50,7 +51,8 @@ class Test::Unit::TestCase
     :status => 200,
     :body_bytes_sent => 6918,
     :http_referer => 'http://www.google.com/search?q=example',
-    :http_user_agent => 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10'
+    :http_user_agent => 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10',
+    :proxy_addresses => nil
   }
   
   REQUEST_FORMAT = '%s %s %s'
@@ -71,6 +73,7 @@ class Test::Unit::TestCase
       options[:body_bytes_sent],
       options[:http_referer],
       options[:http_user_agent],
+      # TODO implement support for :proxy_addresses
     ]
   end