beaver 1.2.0 → 1.3.0

data/lib/beaver/parsers/http.rb

@@ -0,0 +1,109 @@
+module Beaver
+  module Parsers
+    # Parser for HTTP Common Log entries. See the Request class for more log entry attributes.
+    class HTTP < Request
+      # The Combined Log Format
+      FORMAT = '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"' # :nodoc:
+      # The Combined Log Format as an array of symbols
+      FORMAT_SYMBOLS = FORMAT.split(' ').map(&:to_sym) # :nodoc:
+      FHOST, FID, FUSER, FTIME, FREQUEST, FSTATUS, FSIZE, FREFERER, FUA = FORMAT_SYMBOLS # :nodoc:
+
+      # Regex matchers keyed by opening quotes, etc.
+      MATCHERS = {'[' => /^\[([^\]]+)\] ?/, '"' => /^"([^"]+)" ?/} # :nodoc:
+      MATCHERS.default = /^([^ ]+) ?/
+
+      # Matches an HTTP Log entry
+      REGEX_MATCH = %r{\[[0-9]{2}/\w+/[0-9:]+ } # :nodoc:
+      # Matches the request method, url and params
+      REGEX_REQUEST = /^([A-Z]+) (\/[^\?]+)\??(.*) HTTP\/1/ # :nodoc:
+      # Matches the request date
+      REGEX_DATE = %r{^([0-9]{2}/[a-z]+/[0-9]+)}i # :nodoc:
+
+      # Partially parse the request
+      def initialize(data)
+        super
+        pre_parse!
+      end
+
+      # Returns the HTTP method as a symbol
+      def method
+        parse_request! if @method.nil?
+        @method
+      end
+
+      # Returns the request path
+      def path
+        parse_request! if @path.nil?
+        @path
+      end
+
+      # Returns the url query string
+      def params_str
+        parse_request! if @params_str.nil?
+        @params_str
+      end
+
+      # Returns the response size (in bytes)
+      def size
+        @size ||= @request[FSIZE].to_i
+      end
+
+      # Returns the REFERER [sic]
+      def referer
+        @request[FREFERER]
+      end
+      alias_method :referrer, :referer
+
+      # Returns the user agent string
+      def user_agent
+        @request[FUA]
+      end
+
+      protected
+
+      # Parse and cache the request method, URL and any params
+      def parse_request!
+        REGEX_REQUEST.match(@request[FREQUEST])
+        @method = $1 ? $1.downcase.to_sym : :unknown
+        @path = $2 || BLANK_STR
+        @params_str = $3 || BLANK_STR
+      end
+
+      # Parses and returns the response status
+      def parse_status
+        @request[FSTATUS].to_i
+      end
+
+      # Parses and returns the request's IP address
+      def parse_ip
+        @request[FHOST] || BLANK_STR
+      end
+
+      # Parses and returns the time at which the request was made
+      def parse_date
+        REGEX_DATE.match(@request[FTIME]) ? Date.parse($1) : nil
+      end
+
+      # Parses and returns the time at which the request was made
+      def parse_time
+        time_str = @request[FTIME].gsub(/([0-9]{4}):([0-9]{2})/, "#{$1} #{$2}")
+        Time.parse(time_str) rescue nil
+      end
+
+      private
+
+      # Break the log line down into sections, store them in @request as a Hash.
+      # Each section may be further parsed on-demand.
+      # XXX This is probably really slow
+      def pre_parse!
+        line = @data.clone
+        request = {}
+        for f in FORMAT_SYMBOLS
+          line.slice!(MATCHERS[line[0,1]])
+          request[f] = $1
+        end
+        @request = request
+      end
+    end
+  end
+end

data/lib/beaver/parsers/rails.rb

@@ -1,131 +1,108 @@
 module Beaver
   module Parsers
-    # This appears to work with Rails 3 logs
+    # Parser for Rails log entries. See the Request class for more log entry attributes.
     class Rails < Request
-      # Tell the Request class to use this parser to parse logs
-      Request << self
-
-      REGEX_METHOD = /^Started ([A-Z]+)/
-      REGEX_METHOD_OVERRIDE = /"_method"=>"([A-Z]+)"/i
-      REGEX_CONTROLLER = /Processing by (\w+Controller)#/
-      REGEX_ACTION = /Processing by \w+Controller#(\w+) as/
-      REGEX_COMPLETED = /Completed (\d+)/
-      REGEX_PATH = /^Started \w{3,4} "([^"]+)"/
-      REGEX_PARAMS_STR = /  Parameters: (\{.+\})$/
-      REGEX_IP = /" for (\d+[\d.]+) at /
-      REGEX_FORMAT = /Processing by .+ as (\w+)$/
-      REGEX_MS = / in (\d+)ms/
-      REGEX_TAGS = /^(\[.+\] )+/
-      REGEX_TAG = /\[([^\]]+)\] /
+      REGEX_MATCH = /^Started [A-Z]+/ # :nodoc:
+      REGEX_METHOD = /^Started ([A-Z]+)/ # :nodoc:
+      REGEX_METHOD_OVERRIDE = /"_method"=>"([A-Z]+)"/i # :nodoc:
+      REGEX_CONTROLLER = /Processing by (\w+Controller)#/ # :nodoc:
+      REGEX_ACTION = /Processing by \w+Controller#(\w+) as/ # :nodoc:
+      REGEX_COMPLETED = /Completed (\d+)/ # :nodoc:
+      REGEX_PATH = /^Started \w{3,4} "([^"]+)"/ # :nodoc:
+      REGEX_PARAMS_STR = /  Parameters: (\{.+\})$/ # :nodoc:
+      REGEX_IP = /" for ([a-zA-Z0-9:.]+) at / # :nodoc:
+      REGEX_FORMAT = /Processing by .+ as (\w+)$/ # :nodoc:
+      REGEX_MS = / in (\d+)ms/ # :nodoc:
+      REGEX_TAGS = /^(\[.+\] )+/ # :nodoc:
+      REGEX_TAG = /\[([^\]]+)\] / # :nodoc:
       # Depending on the version of Rails, the time format may be wildly different
-      REGEX_TIME = / at ([a-z0-9:\+\- ]+)$/i
+      REGEX_TIME = / at ([a-z0-9:\+\- ]+)$/i # :nodoc:
 
-      # Returns true if the given lines look like a Rails request
-      def self.match?(lines)
-        REGEX_METHOD =~ lines
+      # Returns true if/when we have the final line of the multi-line Rails request
+      def completed?
+        REGEX_COMPLETED =~ @data
       end
 
-      # Returns true, always
-      def valid?; true; end
+      # Returns the class name of the Rails controller that handled the request
+      def controller
+        @controller ||= REGEX_CONTROLLER.match(@data) ? $1 : BLANK_STR
+      end
 
-      # Returns true if/when we have the final line of the multi-line Rails request
-      def completed?
-        REGEX_COMPLETED =~ @lines
+      # Returns the class name of the Rails controller action that handled the request
+      def action
+        @action ||= REGEX_ACTION.match(@data) ? $1.to_sym : :unknown
+      end
+
+      # Returns the responses format (html, json, etc)
+      def format
+        @format ||= REGEX_FORMAT.match(@data) ? $1.downcase.to_sym : :unknown
+      end
+
+      # Returns the number of milliseconds it took for the request to complete
+      def ms
+        @ms ||= REGEX_MS.match(@data) ? $1.to_i : 0
+      end
+
+      # Returns the request parameters as a Hash (this is more expensive than Request#params_str)
+      def params
+        @params ||= params_str.empty? ? BLANK_HASH : Utils.str_to_hash(params_str)
+      end
+
+      # Returns the tags string associated with the request (e.g. "[tag1] [tag2] ")
+      def tags_str
+        @tags_str ||= REGEX_TAGS.match(@data) ? $1 : nil
+      end
+
+      # Returns an array of tags associated with the request
+      def tags
+        @tags ||= if t = tags_str
+          tags = t.scan(REGEX_TAG)
+          tags.flatten!
+          tags.uniq!
+          tags.map! &:downcase
+          tags
+        else
+          []
+        end
       end
 
       protected
 
       # Parses and returns the request path
       def parse_path
-        m = REGEX_PATH.match(@lines)
-        m ? m.captures.first : BLANK_STR
+        REGEX_PATH.match(@data) ? $1 : BLANK_STR
       end
 
       # Parses and returns the request method
       def parse_method
-        m = REGEX_METHOD_OVERRIDE.match(@lines)
-        m = REGEX_METHOD.match(@lines) if m.nil?
+        m = REGEX_METHOD_OVERRIDE.match(@data)
+        m = REGEX_METHOD.match(@data) if m.nil?
         m ? m.captures.first.downcase.to_sym : :unknown
       end
 
-      # Parses the name of the Rails controller which handled the request
-      def parse_controller
-        c = REGEX_CONTROLLER.match(@lines) if c.nil?
-        c ? c.captures.first : BLANK_STR
-      end
-
-      # Parses the name of the Rails controller action which handled the request
-      def parse_action
-        a = REGEX_ACTION.match(@lines) if a.nil?
-        a ? a.captures.first.to_sym : :unknown
-      end
-
       # Parses and returns the response status
       def parse_status
-        m = REGEX_COMPLETED.match(@lines)
-        m ? m.captures.first.to_i : 0
+        REGEX_COMPLETED.match(@data) ? $1.to_i : 0
       end
 
       # Parses and returns the request parameters as a String
       def parse_params_str
-        m = REGEX_PARAMS_STR.match(@lines)
-        m ? m.captures.first : BLANK_STR
-      end
-
-      # Parses and returns the request parameters as a Hash (relatively expensive)
-      def parse_params
-        p = params_str
-        p.empty? ? BLANK_HASH : Utils.str_to_hash(p)
+        REGEX_PARAMS_STR.match(@data) ? $1 : BLANK_STR
       end
 
       # Parses and returns the request's IP address
       def parse_ip
-        m = REGEX_IP.match(@lines)
-        m ? m.captures.first : BLANK_STR
-      end
-
-      # Parses and returns the respones format
-      def parse_format
-        m = REGEX_FORMAT.match(@lines)
-        m ? m.captures.first.to_s.downcase.to_sym : :unknown
-      end
-
-      # Parses and returns the number of milliseconds it took for the request to complete
-      def parse_ms
-        m = REGEX_MS.match(@lines)
-        m ? m.captures.first.to_i : 0
+        REGEX_IP.match(@data) ? $1 : BLANK_STR
       end
 
       # Parses and returns the time at which the request was made
       def parse_date
-        m = REGEX_TIME.match(@lines)
-        m ? Date.parse(m.captures.first) : nil
+        REGEX_TIME.match(@data) ? Date.parse($1) : nil
       end
 
       # Parses and returns the time at which the request was made
       def parse_time
-        m = REGEX_TIME.match(@lines)
-        m ? Time.parse(m.captures.first) : nil
-      end
-
-      # Parses and returns an array of tags string associated with the request
-      def parse_tags_str
-        t = REGEX_TAGS.match(@lines)
-        t ? t.captures.first : nil
-      end
-
-      # Parses and returns an array of tags associated with the request
-      def parse_tags
-        t = tags_str
-        if t
-          tags = t.scan(REGEX_TAG)
-          tags.flatten!
-          tags.uniq!
-          tags.map! &:downcase
-          tags
-        else
-          []
-        end
+        REGEX_TIME.match(@data) ? Time.parse($1) : nil
       end
     end
   end

data/lib/beaver/request.rb

@@ -2,42 +2,43 @@ require 'date'
 require 'time'
 
 module Beaver
-  # Represents a single request from the logs.
+  # Represents a single request from the logs. The base class for Beaver::Parsers::Rails and Beaver::Parsers::HTTP.
+  # Attributes common to both are defined here.
   class Request
-    BLANK_STR = ''
-    BLANK_HASH = {}
+    BLANK_STR = '' # :nodoc:
+    BLANK_HASH = {} # :nodoc:
 
     # Holds the Parser classes used to parser requests
     @types = []
 
-    # Add a child Request parser.
-    def self.<<(type)
-      @types << type
+    # Add a child Request parser
+    def self.inherited(klass)
+      @types << klass
     end
 
-    # Returns the correct Request parser class for the given log lines. If one cannot be found,
-    # the BadRequest class is returned, which the caller will want to ignore.
-    def self.for(lines)
-      @types.select { |t| t.match? lines }.first || BadRequest
+    # Returns a new Request object for the given log line, or nil if one cannot be found.
+    def self.for(line)
+      klass = @types.detect { |t| t.match? line }
+      klass ? klass.new(line) : nil
     end
 
-    # Accepts a String of log lines, presumably ones which belong to a single request.
-    def initialize(lines=nil)
-      @lines = lines || ''
-      @final = false
+    # Returns true if the given line look like a request of this class
+    def self.match?(line)
+      self::REGEX_MATCH =~ line rescue false
     end
 
-    # Returns the log lines that make up this Request.
-    def to_s; @lines; end
+    # Accepts a String of log data, presumably ones which belong to a single request.
+    def initialize(data=nil)
+      @data = data || ''
+      @final = false
+    end
 
-    # Returns true if this is a "good" request.
-    def good?; true; end
-    # Returns true if this is a "bad" request.
-    def bad?; not good?; end
+    # Returns the log data that make up this Request.
+    def to_s; @data; end
 
     # Append a log line
     def <<(line)
-      @lines << line
+      @data << line
     end
 
     # Returns the request path
@@ -50,16 +51,6 @@ module Beaver
       @method ||= parse_method
     end
 
-    # Returns the class name of the Rails controller that handled the request
-    def controller
-      @controller ||= parse_controller
-    end
-
-    # Returns the class name of the Rails controller action that handled the request
-    def action
-      @action ||= parse_action
-    end
-
     # Returns the response status
     def status
       @status ||= parse_status
@@ -70,26 +61,11 @@ module Beaver
       @params_str ||= parse_params_str
     end
 
-    # Returns the request parameters as a Hash (this is more expensive than Request#params_str)
-    def params
-      @params ||= parse_params
-    end
-
     # Returns the IP address of the request
     def ip
       @ip ||= parse_ip
     end
 
-    # Returns the responses format (html, json, etc)
-    def format
-      @format ||= parse_format
-    end
-
-    # Returns the number of milliseconds it took for the request to complete
-    def ms
-      @ms ||= parse_ms
-    end
-
     # Returns the date on which the request was made
     def date
       @date ||= parse_date
@@ -100,16 +76,6 @@ module Beaver
       @time ||= parse_time
     end
 
-    # Returns the tags string associated with the request (e.g. "[tag1] [tag2] ")
-    def tags_str
-      @tags_str ||= parse_tags_str
-    end
-
-    # Returns an array of tags associated with the request
-    def tags
-      @tags ||= parse_tags
-    end
-
     # When called inside of a Beaver::Dam#hit block, this Request will *not* be matched.
     def skip!
       throw :skip
@@ -126,86 +92,29 @@ module Beaver
     end
 
     # Returns true if the request has all the information it needs to be properly parsed
-    def completed?
-      true
-    end
+    def completed?; true; end
 
     protected
 
     # Parses and returns the request path
-    def parse_path
-      BLANK_STR
-    end
+    def parse_path; BLANK_STR; end
 
     # Parses and returns the request method
-    def parse_method
-      :unknown
-    end
-
-    # Parses the name of the Rails controller which handled the request
-    def parse_controller
-      BLANK_STR
-    end
-
-    # Parses the name of the Rails controller action which handled the request
-    def parse_action
-      :unknown
-    end
+    def parse_method; :unknown; end
 
     # Parses and returns the response status
-    def parse_status
-      0
-    end
+    def parse_status; 0; end
 
     # Parses and returns the request params as a String
-    def parse_params_str
-      BLANK_STR
-    end
-
-    # Parses and returns the request params as a Hash
-    def parse_params
-      BLANK_HASH
-    end
+    def parse_params_str; BLANK_STR; end
 
     # Parses and returns the request IP address
-    def parse_ip
-      BLANK_STR
-    end
-
-    # Parses and returns the respones format
-    def parse_format
-      :unknown
-    end
-
-    # Parses and returns the number of milliseconds it took for the request to complete
-    def parse_ms
-      0
-    end
+    def parse_ip; BLANK_STR; end
 
     # Parses and returns the date on which the request was made
-    def parse_date
-      nil
-    end
+    def parse_date; nil; end
 
     # Parses and returns the time at which the request was made
-    def parse_time
-      nil
-    end
-
-    # Parses and returns the tags string associated with the request (e.g. "[tag1] [tag2] ")
-    def parse_tags_str
-      nil
-    end
-
-    # Parses and returns any tags associated with the request
-    def parse_tags
-      nil
-    end
-  end
-
-  # Represents a BadRequest that no parser could figure out.
-  class BadRequest < Request
-    # Returns false, always
-    def good?; false; end
+    def parse_time; nil; end
   end
 end