beaver 1.2.0 → 1.3.0

data/README.rdoc

@@ -1,6 +1,6 @@
 == Beaver, chewing through logs to make something useful
 
-Beaver is a light DSL and command line utility for parsing Rails production logs back into usable data. It answers questions like:
+Beaver is a light DSL and command line utility for parsing HTTP and Rails production logs back into usable data. It answers questions like:
 
 * How many failed logins have there been today? Were they for the same user? From the same IP?
 * How many 500, 404, etc. errors yesterday? On what pages?
@@ -9,14 +9,14 @@ Beaver is a light DSL and command line utility for parsing Rails production logs
 * Rails 3.2 tagged logging is cool, but what's a good way to review them?
 
 Read the full documentation at {jordanhollinger.com/docs/beaver/}[http://jordanhollinger.com/docs/beaver/].
-For a full list of matchers available to "hit", see the Beaver::Dam class. For a full list of methods available inside a "hit" block, or to members
-of the "hits" array in a "dam" block, see the Beaver::Request class.
+For a full list of matchers available to "hit", see the Beaver::Dam class.
+For a full list of methods available inside a "hit" block, or to members of the "hits" array in a "dam" block, see the Beaver::Request, Beaver::Parsers::Rails, and Beaver::Parsers::HTTP classes.
 
 == Installation
   [sudo] gem install beaver
 
-== Run a DSL file with beaver
-  hit :failed_login, :method => :post, :path => '/login', :status => 401
+== Use beaver with a DSL file
+  hit :failed_logins, :method => :post, :path => '/login', :status => 401
 
   hit :new_widgets, :path => '/widgets', :method => :post, :status => 302 do
     puts "A Widget named #{params[:widget][:name]} was created!"
@@ -31,9 +31,9 @@ of the "hits" array in a "dam" block, see the Beaver::Request class.
     puts "user 1 was tagged at #{path} - other tags were: #{tags.join(', ')}"
   end
 
-  hit :server_error, :status => (500..503)
+  hit :errors, :status => (500..503)
 
-  dam :failed_login do
+  dam :failed_logins do
     puts "Failed logins:"
     hits.group_by { |h| h.params[:username] } do |user, fails|
       puts " #{user}"
@@ -43,32 +43,26 @@ of the "hits" array in a "dam" block, see the Beaver::Request class.
     end
   end
 
-  dam :server_errors do
-    puts "Server errors:"
-    hits.group_by(&:status) do |status, errors|
-      puts " There were #{errors.size} #{status} errors"
-    end
+  dam :errors do
+    puts tablize(' | ') { |hit| [hit.status, hit.path, hit.ip, hit.time] }
   end
 
 Run ''beaver'' from the command line, passing in your beaver file and some logs:
 
   beaver my_beaver_file.rb /var/www/rails-app/log/production.log*
 
-== Use your own Beaver
+== Use beaver as a library
   require 'rubygems'
   require 'beaver'
 
-  # Logs from the last day or so
-  logs = ['/var/www/rails-app/log/production.log',
-          '/var/www/rails-app/log/production.log.1']
-
-  # Parse the logs, but only include requests from yesterday
-  Beaver.stream logs, :on => Date.today-1 do
-    hit :failed_login, :method => :post, :path => '/login', :status => 401
-    ...
+  beaver = Beaver.new('/path/to/httpd/access_logs*')
+  beaver.hit :failed_logins, :method => :post, :path => '/login', :status => 401
+  beaver.dam :failed_logins do
+    puts "#{hits.size} failed logins!"
   end
+  beaver.stream
 
-== Beavers love Unix
+== Use beaver as part of the *nix toolchain
 
 It's difficult to grep through a multi-line log format like Rails' and output each matching multi-line event (though I hear Google is working on a 'Context-Free Grep', which may help solve that). Until then, for Rails anyway, beaver is happy to step in.
 
@@ -78,7 +72,7 @@ Or format the output to a single line:
 
   beaver --controller=widgets /var/www/rails-app/log/production.log --print "%{ip} hit %{action} using %{method}"
 
-Also accepts log content from pipes and stdin (might be a bit faster):
+Also accepts log content from pipes and stdin. Use it to filter log files:
 
   cat /var/www/rails-app/log/production.log* | beaver --action=edit
 
@@ -86,6 +80,10 @@ Also accepts log content from pipes and stdin (might be a bit faster):
 
   beaver --action=edit --stdin
 
+Or for dead-simple real-time event monitoring:
+
+  tail -f /var/log/nginx/access_log | beaver www.rb
+
 See all options with 'beaver --help'.
 
 == Example use with Logwatch
@@ -111,7 +109,7 @@ HTTP status codes. For example, your failed logins are probably returning
 
 A detailed description of each status code and when to use it can be found at {www.w3.org}[http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html].
 
-== Complex tag querying
+== Complex Rails tag querying
 Beaver supports complex Rails Tagged Logger tag quering, in both DSL and command-line modes.
 
 === DSL
@@ -128,8 +126,8 @@ Beaver supports complex Rails Tagged Logger tag quering, in both DSL and command
   hit :tagged_with_any, :tagged => [['foo'], ['bar']]
 
   # Matches any request tagged with ("foo" AND "bar") OR ("bar" AND "baz") OR "yay"
-  hit :tagged_and_or_and_or, :tagged => [['foo', 'foop'], ['bar', 'baz'], ['yay']]
-  # Could also be written ['foo, foop', 'bar, baz', 'yay']
+  hit :tagged_and_or_and_or, :tagged => [['foo', 'bar'], ['bar', 'baz'], ['yay']]
+  # Could also be written ['foo, bar', 'bar, baz', 'yay']
 
 === Command-line
 
@@ -142,8 +140,8 @@ Beaver supports complex Rails Tagged Logger tag quering, in both DSL and command
   # Matches any request tagged with "foo" OR "bar"
   beaver --tagged foo --tagged bar production.log
 
-  # Matches any request tagged with ("foo" AND "foop") OR ("bar" AND "baz") OR "yay"
-  beaver --tagged foo,foop --taged bar,baz --tagged yay production.log
+  # Matches any request tagged with ("foo" AND "bar") OR ("bar" AND "baz") OR "yay"
+  beaver --tagged foo,bar --tagged bar,baz --tagged yay production.log
 
 == License
 Copyright 2011 Jordan Hollinger

data/bin/beaver

@@ -6,27 +6,34 @@ require 'beaver'
 
 options, matchers = {}, {}
 o = OptionParser.new do |opts|
-  opts.banner = 'Usage: beaver [options] [dsl.rb...] [/rails/production.log...]'
-  opts.on('--path PATH', 'URL path, string or regex') { |path| matchers[:path] = Regexp.new(path, Regexp::IGNORECASE) }
-  opts.on('--method METHOD', 'HTTP request method(s), e.g. get or post,put') { |methods| matchers[:method] = methods.split(',').map { |m| m.downcase.to_sym } }
-  opts.on('--controller CONTROLLER', 'Rails controller name or regex') { |c| matchers[:controller] = Regexp.new(c, Regexp::IGNORECASE) }
-  opts.on('--action CONTROLLER', 'Rails action name or regex') { |a| matchers[:action] = Regexp.new(a, Regexp::IGNORECASE) }
-  opts.on('--status STATUS', 'HTTP status(es), e.g. 200 or 500..503') { |s| matchers[:status] = s =~ /(\.+)/ ? Range.new(*s.split($1).map(&:to_i)) : s.to_i }
-  opts.on('--ip IP', 'IP address, string or regex') { |ip| matchers[:ip] = Regexp.new(ip) }
-  opts.on('--tagged TAGS', 'Comma-separated Rails Tagged Logger tags') { |tags| (matchers[:tagged] ||= []) << tags }
-  opts.on('--params PARAMS', 'Request parameters string (a Ruby Hash), string or regex') { |params| matchers[:params_str] = Regexp.new(params, Regexp::IGNORECASE) }
-  opts.on('--format FORMAT', 'Response format(s), e.g. html or json,xml') { |f| matchers[:format] = f.split(',').map(&:to_sym) }
-  opts.on('--longer-than MS', 'Minimum response time in ms') { |ms| matchers[:longer_than] = ms.to_i }
-  opts.on('--shorter-than MS', 'Maximum response time in ms') { |ms| matchers[:shorter_than] = ms.to_i }
-  opts.on('--on DATE', 'Only include log entries from the given date (yyyy-mm-dd or -n days)') { |d| matchers[:on] = Beaver::Utils.parse_date(d) }
-  opts.on('--after DATE', 'Only include log entries from after the given date (yyyy-mm-dd or -n days)') { |d| matchers[:after] = Beaver::Utils.parse_date(d) }
-  opts.on('--before DATE', 'Only include log entries from before the given date (yyyy-mm-dd or -n days)') { |d| matchers[:before] = Beaver::Utils.parse_date(d) }
-  opts.on('--today', 'Alias to --on=-0') { matchers[:on] = Date.today }
-  opts.on('--yesterday', 'Alias to --on=-1') { matchers[:on] = Date.today-1 }
-  opts.on('--regex REGEX', 'A regex string to be matched against the entire request') { |r| matchers[:match] = Regexp.new(r, Regexp::IGNORECASE) }
+  opts.banner = 'Usage: beaver [options] [dsl.rb...] [/path/to/log...]'
+  opts.on('--path PATH', 'Rails HTTP  URL path, string or regex') { |path| matchers[:path] = Regexp.new(path, Regexp::IGNORECASE) }
+  opts.on('--method METHOD', 'Rails HTTP  HTTP request method(s), e.g. get or post,put') { |methods| matchers[:method] = methods.split(',').map { |m| m.downcase.to_sym } }
+  opts.on('--controller CONTROLLER', 'Rails       Controller name or regex') { |c| matchers[:controller] = Regexp.new(c, Regexp::IGNORECASE) }
+  opts.on('--action ACTION', 'Rails       Action name or regex') { |a| matchers[:action] = Regexp.new(a, Regexp::IGNORECASE) }
+  opts.on('--status STATUS', 'Rails HTTP  HTTP status(es), e.g. 200 or 500..503') { |s| matchers[:status] = s =~ /(\.+)/ ? Range.new(*s.split($1).map(&:to_i)) : s.to_i }
+  opts.on('--ip IP', 'Rails HTTP  IP address, string or regex') { |ip| matchers[:ip] = Regexp.new(ip) }
+  opts.on('--tagged TAGS', 'Rails       Comma-separated Rails Tagged Logger tags') { |tags| (matchers[:tagged] ||= []) << tags }
+  opts.on('--params PARAMS', 'Rails HTTP  Request parameters string (a Ruby Hash), string or regex') { |params| matchers[:params_str] = Regexp.new(params, Regexp::IGNORECASE) }
+  opts.on('--format FORMAT', 'Rails       Response format(s), e.g. html or json,xml') { |f| matchers[:format] = f.split(',').map(&:to_sym) }
+  opts.on('--longer-than MS', 'Rails       Minimum response time in ms') { |ms| matchers[:longer_than] = ms.to_i }
+  opts.on('--shorter-than MS', 'Rails       Maximum response time in ms') { |ms| matchers[:shorter_than] = ms.to_i }
+  opts.on('--on DATE', 'Rails HTTP  Only include log entries from the given date (yyyy-mm-dd or -n days)') { |d| matchers[:on] = Beaver::Utils.parse_date(d) }
+  opts.on('--after DATE', 'Rails HTTP  Only include log entries from after the given date (yyyy-mm-dd or -n days)') { |d| matchers[:after] = Beaver::Utils.parse_date(d) }
+  opts.on('--before DATE', 'Rails HTTP  Only include log entries from before the given date (yyyy-mm-dd or -n days)') { |d| matchers[:before] = Beaver::Utils.parse_date(d) }
+  opts.on('--today', 'Rails HTTP  Alias to --on=-0') { matchers[:on] = Date.today }
+  opts.on('--yesterday', 'Rails HTTP  Alias to --on=-1') { matchers[:on] = Date.today-1 }
+  opts.on('--size BYTES', '      HTTP  Responses of n bytes') { |bytes| matchers[:size] = bytes.to_i }
+  opts.on('--smaller-than BYTES', '      HTTP  Responses smaller than n bytes') { |bytes| matchers[:smaller_than] = bytes.to_i }
+  opts.on('--larger-than BYTES', '      HTTP  Responses larger than n bytes') { |bytes| matchers[:larger_than] = bytes.to_i }
+  opts.on('--referer URL', '      HTTP  Referer URL, string or regex') { |url| matchers[:referer] = Regexp.new(url, Regexp::IGNORECASE) }
+  opts.on('--referrer URL', '      HTTP  Referer URL, string or regex') { |url| matchers[:referrer] = Regexp.new(url, Regexp::IGNORECASE) }
+  opts.on('--user-agent STRING', '      HTTP  User Agent, string or regex') { |ua| matchers[:user_agent] = Regexp.new(ua, Regexp::IGNORECASE) }
+  opts.on('--regex REGEX', 'Rails HTTP  A regex string to be matched against the entire request') { |r| matchers[:match] = Regexp.new(r, Regexp::IGNORECASE) }
   opts.on('--print FORMAT', 'Formatted request string, e.g. "%{ip} went to %{path} passing %{params[:email]}"') { |hit| options[:print] = hit }
+  opts.on('--tablize', 'Print output from --print in a table') { options[:tablize] = true }
   opts.on('--stdin', 'Read log content from stdin (for typing/pasting)') { options[:tty] = true }
-  opts.on('-v', '--version', 'Show version') { puts "beaver #{Beaver::VERSION}"; exit }
+  opts.on('-v', '--version', 'Print version and exit') { puts "beaver #{Beaver::VERSION}"; exit }
 end
 o.parse!
 
@@ -40,12 +47,36 @@ Beaver.stream *args do
   tty! if options[:tty]
   # Filter the logs through the DSL files
   if beaver_files.any?
-    beaver_files.map! { |b| File.open(b, &:read) }
-    beaver_files.each { |b| eval b }
+    beaver_files.map { |b| File.open(b, &:read) }.each &method(:eval)
   # Filter the logs through the CLI options
   else
-    hit :cli, matchers do
-      puts options[:print] ? eval('"'+options[:print].gsub(/%\{/, '#{')+'"') : to_s << $/
+    # Format each entry
+    if options[:print]
+      options[:print].gsub!(/%\{/, '#{')
+      arg_pattern = /#\{[^\}]+\}/
+
+      # Make each column the same length
+      if options[:tablize]
+        hit :cli, matchers
+        dam :cli do
+          text = options[:print].split(arg_pattern) << ''
+          tablize(false) do |hit|
+            options[:print].gsub(arg_pattern).map { |match| hit.instance_eval(%Q|"#{match}"|) rescue '?' }
+          end.each do |cols|
+            puts text.map { |t| "#{t}#{cols.shift}" }.join
+          end
+        end
+      # Just format and print
+      else
+        hit :cli, matchers do
+          puts options[:print].gsub(arg_pattern) { |match| self.instance_eval(%Q|"#{match}"|) rescue '?' }
+        end
+      end
+    # Print the entirety of each entry
+    else
+      hit :cli, matchers do
+        puts to_s
+      end
     end
   end
 end

data/lib/beaver.rb

@@ -4,9 +4,11 @@ rescue LoadError
   $stderr.puts "Zlib not available; compressed log files will be skipped."
 end
 
+require 'beaver/version'
 require 'beaver/utils'
 require 'beaver/beaver'
 require 'beaver/dam'
 require 'beaver/request'
 
 require 'beaver/parsers/rails'
+require 'beaver/parsers/http'

data/lib/beaver/beaver.rb

@@ -1,16 +1,14 @@
 # Not specifically a performance analyzer (like https://github.com/wvanbergen/request-log-analyzer/wiki)
 # Rather, a DSL for finding out how people are using your Rails app (which could include performance).
+# Can also be used to parse/analyze HTTP access logs (Apache, Nginx, etc.)
 # 
-# Beaver.stream do
-#   hit :error, :status => (400..505) do
-#     puts "#{status} on #{path} at #{time} from #{ip} with #{params_str}"
-#   end
-# end
+#  Beaver.stream('/path/to/log/files') do
+#    hit :error, :status => (400..505) do
+#      puts "#{status} on #{path} at #{time} from #{ip} with #{params_str}"
+#    end
+#  end
 # 
 module Beaver
-  MAJOR_VERSION, MINOR_VERSION, TINY_VERSION, PRE_VERSION = 1, 2, 0, nil
-  VERSION = [MAJOR_VERSION, MINOR_VERSION, TINY_VERSION, PRE_VERSION].compact.join '.'
-
   # Creates a new Beaver and immediately filters the log files. This should scale well
   # for even very large logs, at least when compared to Beaver#parse.
   def self.stream(*args, &blk)
@@ -25,7 +23,7 @@ module Beaver
     Beaver.new(*args, &blk).parse.filter
   end
 
-  # Alias to creating a new Beaver
+  # Alias to Beaver::Beaver.new
   def self.new(*args, &blk)
     Beaver.new(*args, &blk)
   end
@@ -33,18 +31,18 @@ module Beaver
   # The Beaver class, which keeps track of the files you're parsing, the Beaver::Dam objects you've defined,
   # and parses and filters the matching Beaver::Request objects.
   # 
-  # beaver = Beaver.new do
-  #   hit :help, :path => '/help' do
-  #     puts "#{ip} needed help"
-  #   end
-  # end
+  #  beaver = Beaver.new do
+  #    hit :help, :path => '/help' do
+  #      puts "#{ip} needed help"
+  #    end
+  #  end
   # 
-  # # Method 1 - logs will be parsed and filtered line-by-line, then discarded. Performance should be constant regardless of the number of logs.
-  # beaver.stream
+  #  # Method 1 - logs will be parsed and filtered line-by-line, then discarded. Performance should be constant regardless of the number of logs.
+  #  beaver.stream
   # 
-  # # Method 2 - all of the logs will be parsed at once and stored in "beaver.requests". Then each request will be filtered.
-  # # This does not scale as well, but is necessary *if you want to hang onto the parsed requests*.
-  # beaver.parse.filter
+  #  # Method 2 - all of the logs will be parsed at once and stored in "beaver.requests". Then each request will be filtered.
+  #  # This does not scale as well, but is necessary *if you want to hang onto the parsed requests*.
+  #  beaver.parse.filter
   # 
   class Beaver
     # The log files to parse
@@ -72,15 +70,17 @@ module Beaver
     # Creates a new Dam and appends it to this Beaver. name should be a unique symbol.
     # See Beaver::Dam for available options.
     def hit(dam_name, matchers={}, &callback)
-      STDERR.puts "WARNING Overwriting existing hit '#{dam_name}'" if @dams.has_key? dam_name
+      $stderr.puts "WARNING Overwriting existing hit '#{dam_name}'" if @dams.has_key? dam_name
       matchers = @global_matchers.merge matchers
       @dams[dam_name] = Dam.new(dam_name, matchers, &callback)
     end
 
     # Define a sumarry for a Dam
-    def dam(name, &callback)
-      STDERR.puts "WARNING Overwriting existing dam '#{name}'" if @sums.has_key? name
+    def dam(name, hit_options={}, &callback)
+      $stderr.puts "WARNING Overwriting existing dam '#{name}'" if @sums.has_key? name
       @sums[name] = callback
+      # Optionally create a new hit
+      hit(name, hit_options) if @dams[name].nil?
     end
 
     # Parses the logs and immediately filters them through the dams. Requests are not retained,
@@ -140,7 +140,7 @@ module Beaver
     # Run the callback on each dam matching request. Optionally pass a block, which will be passed back matching dams.
     def hit_dams(request, &blk)
       for dam in @dams.values
-        if dam.matches? request
+        if dam === request
           catch :skip do
             request.instance_eval(&dam.callback) if dam.callback
             blk.call(dam) if block_given?
@@ -158,7 +158,7 @@ module Beaver
             blk.call(@dams[dam_name]) if block_given?
           end
         else
-          STDERR.puts "WARNING You have defined a dam for '#{dam_name}', but there is no hit defined for '#{dam_name}'"
+          $stderr.puts "WARNING You have defined a dam for '#{dam_name}', but there is no hit defined for '#{dam_name}'"
         end
       end
     end
@@ -168,12 +168,12 @@ module Beaver
       request = nil
       # Parses a line into part of a request
       parse_it = lambda { |line|
-        request ||= Request.for(line).new
-        if request.bad?
-          request = nil
-          next
+        if request
+          request << line
+        else
+          request = Request.for(line)
+          next if request.nil?
         end
-        request << line
         if request.completed?
           blk.call(request)
           request = nil
@@ -184,7 +184,11 @@ module Beaver
       if @tty
         STDIN.read.each_line &parse_it # Read entire stream, then parse it - looks much better to the user
       elsif !STDIN.tty?
-        STDIN.each_line &parse_it
+        begin
+          STDIN.each_line &parse_it
+        rescue Interrupt
+          $stderr.puts 'Closing input stream; parsing input...'
+        end
       end if @stdin
       request = nil
 

data/lib/beaver/dam.rb

@@ -2,43 +2,59 @@ module Beaver
   # A Dam "traps" certain Requests, using one or more matching options. A request must meet *all* of the 
   # matching options specified.
   # 
-  # Matchers:
+  # The last argument may be a block, which will be called everytime this Dam is hit.
+  # The block will be run in the context of the Request object. This can be used for 
+  # further checks or for reporting purposes.
+  # 
+  #  hit :reads, :method => :get do
+  #    puts "#{ip} read from #{path} at #{time}"
+  #  end
+  # 
+  # Available matchers:
   #
-  #  :path => String for exact match, or Regex
+  #  :path          Rails HTTP  String for exact match, or Regex
   #
-  #  :method => A Symbol of :get, :post, :put or :delete, or any array of any (reads the magic _method field if present)
+  #  :method        Rails HTTP  Symbol of :get, :post, :put or :delete, or any array of any (reads the magic _method field if present)
   # 
-  #  :controller => A String like 'FooController' or a Regex like /foo/i
+  #  :controller    Rails       String like 'FooController' or a Regex like /foo/i
   # 
-  #  :action => A String like 'index' or a Regex like /(index)|(show)/
+  #  :action        Rails       String like 'index' or a Regex like /(index)|(show)/
   #
-  #  :status => A Fixnum like 404 or a Range like (500..503)
+  #  :status        Rails HTTP  Fixnum like 404 or a Range like (500..503)
   # 
-  #  :ip => String for exact match, or Regex
+  #  :ip            Rails HTTP  String for exact match, or Regex
   #
-  #  :format => A symbol or array of symbols of response formats like :html, :json
+  #  :format        Rails       Symbol or array of symbols of response formats like :html, :json
   # 
-  #  :longer_than => Fixnum n. Matches any request which took longer than n milliseconds to complete.
+  #  :longer_than   Rails       Fixnum n. Matches any request which took longer than n milliseconds to complete.
   # 
-  #  :shorter_than => Fixnum n. Matches any request which took less than n milliseconds to complete.
+  #  :shorter_than  Rails       Fixnum n. Matches any request which took less than n milliseconds to complete.
   # 
-  #  :before => Date or Time for which the request must have ocurred before
+  #  :before        Rails HTTP  Date or Time for which the request must have ocurred before
   # 
-  #  :after => Date or Time for which the request must have ocurred after
+  #  :after         Rails HTTP  Date or Time for which the request must have ocurred after
   # 
-  #  :on => Date - the request must have ocurred on this date
+  #  :on            Rails HTTP  Date - the request must have ocurred on this date
   # 
-  #  :params_str => A regular expressing matching the Parameters string
+  #  :params_str    Rails HTTP  Regular expressing matching the Parameters string
   # 
-  #  :params => A Hash of Symbol=>String/Regexp pairs: {:username => 'bob', :email => /@gmail\.com$/}. All must match.
+  #  :params        Rails       Hash of Symbol=>String/Regexp pairs: {:username => 'bob', :email => /@gmail\.com$/}. All must match.
   # 
-  #  :tagged => A comma-separated String or Array of Rails Tagged Logger tags. If you specify multiple tags, a request must have *all* of them.
+  #  :tagged        Rails       Comma-separated String or Array of Rails Tagged Logger tags. If you specify multiple tags, a request must have *all* of them.
   # 
-  #  :match => A "catch-all" Regex that will be matched against the entire request string
+  #  :size                HTTP  Fixnum matching the response size in bytes
   # 
-  # The last argument may be a block, which will be called everytime this Dam is hit.
-  # The block will be run in the context of the Request object. This can be used for 
-  # further checks or for reporting purposes.
+  #  :smaller_than        HTTP  Fixnum matching the maximum response size in bytes
+  # 
+  #  :larger_than         HTTP  Fixnum matching the minimum response size in bytes
+  # 
+  #  :referer             HTTP  String for exact match, or Regex
+  # 
+  #  :referrer            HTTP  Alias to :referer
+  # 
+  #  :user_agent          HTTP  String for exact match, or Regex
+  # 
+  #  :match         Rails HTTP  A "catch-all" Regex that will be matched against the entire request string
   class Dam
     # The symbol name of this Beaver::Dam
     attr_reader :name
@@ -56,15 +72,30 @@ module Beaver
       build matchers
     end
 
+    # Transforms arrays of values into rows with equally padded columns. 
+    # Useful for generating table-like formatting of hits.
+    # If delim is falsey, the columns will not be joined, but returned as arrays.
+    # 
+    #  dam :errors do
+    #    puts tablize { |hit| [hit.ip, hit.path, hit.status] }
+    #  end
+    def tablize(delim=' ', &block)
+      rows = Utils.tablize(hits.map &block)
+      rows.map! { |cols| cols.join(delim) } if delim
+      rows
+    end
+
     # Returns an array of IP address that hit this Dam.
     def ips
       @ips ||= @hits.map(&:ip).uniq
     end
 
     # Returns true if the given Request hits this Dam, false if not.
-    def matches?(request)
+    def ===(request)
       return false if request.final?
       return false unless @match_path.nil? or @match_path === request.path
+      return false unless @match_referer.nil? or @match_referer === request.referer
+      return false unless @match_user_agent.nil? or @match_user_agent === request.user_agent
       return false unless @match_longer.nil? or @match_longer < request.ms
       return false unless @match_shorter.nil? or @match_shorter > request.ms
       return false unless @match_method_s.nil? or @match_method_s == request.method
@@ -75,153 +106,165 @@ module Beaver
       return false unless @match_ip.nil? or @match_ip === request.ip
       return false unless @match_format_s.nil? or @match_format_s == request.format
       return false unless @match_format_a.nil? or @match_format_a.include? request.format
-      return false unless @match_before.nil? or @match_before > request.time
-      return false unless @match_after.nil? or @match_after < request.time
+      return false unless @match_before_time.nil? or @match_before_time > request.time
+      return false unless @match_before_date.nil? or @match_before_date > request.date
+      return false unless @match_after_time.nil? or @match_after_time < request.time
+      return false unless @match_after_date.nil? or @match_after_date < request.date
       return false unless @match_on.nil? or @match_on == request.date
       return false unless @match_params_str.nil? or @match_params_str =~ request.params_str
+      return false unless @match_size.nil? or @match_size == request.size
+      return false unless @match_size_lt.nil? or request.size < @match_size_lt
+      return false unless @match_size_gt.nil? or request.size > @match_size_gt
       return false unless @match_r.nil? or @match_r =~ request.to_s
       if @deep_tag_match
-        return false unless @match_tags.nil? or (@match_tags.any? and request.tags_str and deep_matching_tags(@match_tags, request.tags))
+        return false unless @match_tags.nil? or (@match_tags.any? and request.tags_str and Utils.deep_matching_tags(@match_tags, request.tags))
       else
         return false unless @match_tags.nil? or (@match_tags.any? and request.tags_str and (@match_tags - request.tags).empty?)
       end
-      return false unless @match_params.nil? or matching_hashes?(@match_params, request.params)
+      return false unless @match_params.nil? or Utils.matching_hashes?(@match_params, request.params)
       return true
     end
 
-    private
-
-    # Matches tags recursively
-    def deep_matching_tags(matchers, tags)
-      all_tags_matched = nil
-      any_arrays_matched = false
-      for m in matchers
-        if m.is_a? Array
-          matched = deep_matching_tags m, tags
-          any_arrays_matched = true if matched
-        else
-          matched = tags.include? m
-          all_tags_matched = (matched && all_tags_matched != false) ? true : false
-        end
-      end
-      return (all_tags_matched or any_arrays_matched)
-    end
-
-    # Recursively compares to Hashes. If all of Hash A is in Hash B, they match.
-    def matching_hashes?(a,b)
-      intersecting_keys = a.keys & b.keys
-      if intersecting_keys.any?
-        a_values = a.values_at(*intersecting_keys)
-        b_values = b.values_at(*intersecting_keys)
-        indicies = (0..b_values.size-1)
-        indicies.all? do |i|
-          if a_values[i].is_a? String
-            a_values[i] == b_values[i]
-          elsif a_values[i].is_a?(Regexp) and b_values[i].is_a?(String)
-            a_values[i] =~ b_values[i]
-          elsif a_values[i].is_a?(Hash) and b_values[i].is_a?(Hash)
-            matching_hashes? a_values[i], b_values[i]
-          else
-            false
-          end
-        end
-      else
-        false
-      end
-    end
-
-    public
-
     # Parses and checks the validity of the matching options passed to the Dam.
+    # XXX Yikes this is long and ugly...
     def build(matchers)
+      # Match path
       if matchers[:path].respond_to? :===
         @match_path = matchers[:path]
       else
         raise ArgumentError, "Path must respond to the '===' method; try a String or a Regexp (it's a #{matchers[:path].class.name})"
       end if matchers[:path]
 
+      # Match HTTP referer
+      referer = matchers[:referer] || matchers[:referrer]
+      if referer.respond_to? :===
+        @match_referer = referer
+      else
+        raise ArgumentError, "Referrer must respond to the '===' method; try a String or a Regexp (it's a #{referer.class.name})"
+      end if referer
+
+      # Match request method
       case
         when matchers[:method].is_a?(Symbol) then @match_method_s = matchers[:method].to_s.downcase.to_sym
         when matchers[:method].is_a?(Array) then @match_method_a = matchers[:method].map { |m| m.to_s.downcase.to_sym }
         else raise ArgumentError, "Method must be a Symbol or an Array (it's a #{matchers[:method].class.name})"
       end if matchers[:method]
 
+      # Match Rails controller
       if matchers[:controller].respond_to? :===
         @match_controller = matchers[:controller]
       else
         raise ArgumentError, "Controller must respond to the '===' method; try a String or a Regexp (it's a #{matchers[:controller].class.name})"
       end if matchers[:controller]
 
+      # Match Rails controller action
       if matchers[:action].respond_to? :=== or matchers[:action].is_a? Symbol
         @match_action = matchers[:action]
       else
         raise ArgumentError, "Action must respond to the '===' method or be a Symbol; try a String, Symbol or a Regexp (it's a #{matchers[:action].class.name})"
       end if matchers[:action]
 
+      # Match response status
       case matchers[:status].class.name
         when Fixnum.name, Range.name then @match_status = matchers[:status]
         else raise ArgumentError, "Status must be a Fixnum or a Range (it's a #{matchers[:status].class.name})"
       end if matchers[:status]
 
+      # Match request IP
       if matchers[:ip].respond_to? :===
         @match_ip = matchers[:ip]
       else
         raise ArgumentError, "IP must respond to the '===' method; try a String or a Regexp (it's a #{matchers[:ip].class.name})"
       end if matchers[:ip]
 
+      # Match Rails' response format
       case
         when matchers[:format].is_a?(Symbol) then @match_format_s = matchers[:format].to_s.downcase.to_sym
         when matchers[:format].is_a?(Array) then @match_format_a = matchers[:format].map { |f| f.to_s.downcase.to_sym }
         else raise ArgumentError, "Format must be a Symbol or an Array (it's a #{matchers[:format].class.name})"
       end if matchers[:format]
 
+      # Match Rails' response time (at least)
       case matchers[:longer_than].class.name
         when Fixnum.name then @match_longer = matchers[:longer_than]
         else raise ArgumentError, "longer_than must be a Fixnum (it's a #{matchers[:longer_than].class.name})"
       end if matchers[:longer_than]
 
+      # Match Rails' response time (at most)
       case matchers[:shorter_than].class.name
         when Fixnum.name then @match_shorter = matchers[:shorter_than]
         else raise ArgumentError, "shorter_than must be a Fixnum (it's a #{matchers[:shorter_than].class.name})"
       end if matchers[:shorter_than]
 
-      @match_before = if matchers[:before].is_a? Time
-        matchers[:before]
+      # Match HTTP response size
+      case matchers[:size].class.name
+        when Fixnum.name then @match_size = matchers[:size]
+        else raise ArgumentError, "size must be a Fixnum (it's a #{matchers[:size].class.name})"
+      end if matchers[:size]
+
+      # Match HTTP response size (at most)
+      case matchers[:smaller_than].class.name
+        when Fixnum.name then @match_size_lt = matchers[:smaller_than]
+        else raise ArgumentError, "size must be a Fixnum (it's a #{matchers[:smaller_than].class.name})"
+      end if matchers[:smaller_than]
+
+      # Match HTTP response size (at least)
+      case matchers[:larger_than].class.name
+        when Fixnum.name then @match_size_gt = matchers[:larger_than]
+        else raise ArgumentError, "size must be a Fixnum (it's a #{matchers[:larger_than].class.name})"
+      end if matchers[:larger_than]
+
+      # Match before a request date
+      if matchers[:before].is_a? Time
+        @match_before_time = matchers[:before]
       elsif matchers[:before].is_a? Date
-        Utils::NormalizedTime.new(matchers[:before].year, matchers[:before].month, matchers[:before].day)
+        @match_before_date = matchers[:before]
       else
         raise ArgumentError, "before must be a Date or Time (it's a #{matchers[:before].class.name})"
       end if matchers[:before]
 
-      @match_after = if matchers[:after].is_a? Time
-        matchers[:after]
+      # Match after a request date or datetime
+      if matchers[:after].is_a? Time
+        @match_after_time = matchers[:after]
       elsif matchers[:after].is_a? Date
-        Utils::NormalizedTime.new(matchers[:after].year, matchers[:after].month, matchers[:after].day)
+        @match_after_date = matchers[:after]
       else
         raise ArgumentError, "after must be a Date or Time (it's a #{matchers[:after].class.name})"
       end if matchers[:after]
 
+      # Match a request date
       if matchers[:on].is_a? Date
         @match_on = matchers[:on]
       else
         raise ArgumentError, "on must be a Date (it's a #{matchers[:on].class.name})"
       end if matchers[:on]
 
+      # Match request URL parameters string
       case matchers[:params_str].class.name
         when Regexp.name then @match_params_str = matchers[:params_str]
         else raise ArgumentError, "Params String must be a Regexp (it's a #{matchers[:params_str].class.name})"
       end if matchers[:params_str]
 
+      # Match request URL parameters Hash
       case matchers[:params].class.name
         when Hash.name then @match_params = matchers[:params]
         else raise ArgumentError, "Params must be a String or a Regexp (it's a #{matchers[:params].class.name})"
       end if matchers[:params]
 
+      # Match Rails request tags
       if matchers[:tagged]
         @match_tags = parse_tag_matchers(matchers[:tagged])
         @deep_tag_match = @match_tags.any? { |t| t.is_a? Array }
       end
 
+      # Match HTTP user agent string
+      if matchers[:user_agent].respond_to? :===
+        @match_user_agent = matchers[:user_agent]
+      else
+        raise ArgumentError, "User Agent must respond to the '===' method; try a String or a Regexp (it's a #{matchers[:user_agent].class.name})"
+      end if matchers[:user_agent]
+
+      # Match the entire log entry string
       case matchers[:match].class.name
         when Regexp.name then @match_r = matchers[:match]
         else raise ArgumentError, "Match must be a Regexp (it's a #{matchers[:match].class.name})"