apachecrunch 0.4 → 0.5

data/bin/apachecrunch

@@ -66,5 +66,5 @@ log_parser = ApacheCrunch::LogParserFactory.log_parser(
                         format_def,
                         options[:logfile],
                         progress_meter)
-proc_env = ProcedureEnvironment.new(log_parser)
+proc_env = ApacheCrunch::ProcedureEnvironment.new(log_parser)
 proc_env.eval_procedure(open(options[:procedure]).read())

data/lib/apachecrunch.rb

@@ -2,21 +2,11 @@ require "date"
 require "tempfile"
 
 require 'config'
+require 'progress'
 require 'entry'
 require 'format'
+require 'derivation'
+require 'format_token'
 require 'log_parser'
-require 'log_element'
-
-class ApacheCrunch
-    # A bare string in a log format
-    #
-    # Exposes 'regex' for consistency with LogFormatElement, but there shouldn't be anything other
-    # than one-to-one character matching in there.
-    class LogFormatString
-        attr_accessor :regex
-
-        def initialize(regex)
-            @regex = regex
-        end
-    end
-end
+require 'element'
+require 'element_value_fetcher'

data/lib/cast.rb

@@ -0,0 +1,21 @@
+class ApacheCrunch
+    # Converts a string to an integer
+    class IntegerCast
+        def cast(string_value)
+            string_value.to_i
+        end
+    end
+
+    # Converts a CLF-formatted string to an integer
+    #
+    # "CLF-formatted" means that if the value is 0, the string will be a single hyphen instead of
+    # a number.  Like %b, for instance.
+    class CLFIntegerCast
+        def cast(string_value)
+            if string_value == "-"
+                return 0
+            end
+            string_value.to_i
+        end
+    end
+end

data/lib/derivation.rb

@@ -0,0 +1,113 @@
+class ApacheCrunch
+    # Abstract for classes that define how to obtain a given element from the value of another.
+    class DerivationRule
+        # Returns the name of the element from which this rule derives values
+        def source_name
+            raise NotImplementedError
+        end
+
+        # Derives the given derivable element from the given element value
+        def derive(name, source_value)
+            raise NotImplementedError
+        end
+    end
+
+
+    # Dummy rule that doesn't derive anything
+    class NullDerivationRule
+        def source_name; nil; end
+        def target_names; []; end
+        def derive(name, source_value); nil; end
+    end
+
+
+    # Derivation rule for elements derived from TimeToken
+    class TimeDerivationRule < DerivationRule
+        def initialize
+            @_derivation_regex = nil
+            @_month_map = {"Jan" => 1, "Feb" => 2, "Mar" => 3, "Apr" => 4,
+                           "May" => 5, "Jun" => 6, "Jul" => 7, "Aug" => 8,
+                           "Sep" => 9, "Oct" => 10, "Nov" => 11, "Dec" => 12}
+        end
+
+        def source_name
+            :time
+        end
+
+        def target_names
+            [:year, :month, :day, :hour, :minute, :second]
+        end
+
+        def derive(name, source_value)
+            if @_derivation_regex.nil?
+                @_derivation_regex = Regexp.compile(%q!^\[(\d\d)/([A-Za-z]{3})/(\d\d\d\d):(\d\d):(\d\d):(\d\d)!)
+            end
+
+            hsh = {}
+            if source_value =~ @_derivation_regex
+                hsh[:year] = $3.to_i
+                hsh[:month] = @_month_map[$2]
+                hsh[:day] = $1.to_i
+
+                hsh[:hour] = $4.to_i
+                hsh[:minute] = $5.to_i
+                hsh[:second] = $6.to_i
+            end
+
+            hsh[name]
+        end
+    end
+
+    class ReqFirstlineDerivationRule
+        def initialize
+            @_derivation_regex = nil
+        end
+
+        def source_name
+            :req_firstline
+        end
+
+        def target_names
+            [ReqMethodTokenDefinition.name, UrlPathTokenDefinition.name, QueryStringTokenDefinition.name, ProtocolTokenDefinition.name]
+        end
+
+        def derive(name, source_value)
+            if @_derivation_regex.nil?
+                @_derivation_regex = Regexp.compile("^(#{ReqMethodTokenDefinition.regex})\s+(#{UrlPathTokenDefinition.regex})(#{QueryStringTokenDefinition.regex})\s+(#{ProtocolTokenDefinition.regex})$")
+            end
+
+            hsh = {}
+            if source_value =~ @_derivation_regex
+                hsh[ReqMethodTokenDefinition.name] = $1
+                hsh[UrlPathTokenDefinition.name] = $2
+                hsh[QueryStringTokenDefinition.name] = $3
+                hsh[ProtocolTokenDefinition.name] = $4
+            end
+
+            hsh[name]
+        end
+    end
+
+    class DerivationRuleFinder
+        @_rule_map = nil
+        @_rules = [NullDerivationRule, TimeDerivationRule, ReqFirstlineDerivationRule]
+
+        # Returns a derivation rule that derives element with the given name
+        def self.find(element_name)
+            @_rule_map = self._build_rule_map if @_rule_map.nil?
+            @_rule_map[element_name]
+        end
+
+        def self._build_rule_map
+            hsh = {}
+            @_rules.each do |rule_cls|
+                r = rule_cls.new
+                r.target_names.each do |target_element|
+                    hsh[target_element] = r
+                end
+            end
+
+            hsh
+        end
+    end
+end

data/lib/element.rb

@@ -0,0 +1,16 @@
+class ApacheCrunch
+    class Element
+        attr_accessor :token, :value
+
+        def populate!(token, value)
+            @token = token
+            @value = value
+        end
+
+        def name; @token.name; end
+
+        def derivation_rule
+            @token.derivation_rule
+        end
+    end
+end

data/lib/element_value_fetcher.rb

@@ -0,0 +1,72 @@
+class ApacheCrunch
+    # Finds a value from an Entry.  Value may be directly from log or derived.
+    class ElementValueFetcher
+        def initialize
+            @_RawValueFetcher = RawValueFetcher
+            @_DerivedValueFetcher = DerivedValueFetcher
+        end
+
+        # Handles dependency injection
+        def dep_inject!(raw_value_fetcher_cls, derived_value_fetcher_cls)
+            @_RawValueFetcher = raw_value_fetcher_cls
+            @_DerivedValueFetcher = derived_value_fetcher_cls
+        end
+
+        # Returns the value of the element with the given name from the Entry instance.
+        #
+        # So element_name might be :minute or :reqheader_firstline for instance.
+        def fetch(entry, element_name)
+            v = @_RawValueFetcher.new.fetch(entry, element_name)
+            return v unless v.nil?
+
+            v = @_DerivedValueFetcher.new.fetch(entry, element_name)
+            return v unless v.nil?
+
+            nil
+        end
+    end
+
+
+    # Returns the value of an element that was captured straight from the log.
+    class RawValueFetcher
+        # Returns the value of the Element with the given name in the given Entry.
+        #
+        # Only works for elements based on tokens that we parsed directly into the Entry.  If no
+        # matching element is found, we return nil.
+        def fetch(entry, element_name)
+            element = entry.captured_elements[element_name]
+            return nil if element.nil?
+            element.value
+        end
+    end
+
+
+    # Returns the value of an element derived from one captured directly from the log.
+    class DerivedValueFetcher
+        def initialize
+            @_DerivationRuleFinder = DerivationRuleFinder
+        end
+
+        # Handles dependency injection
+        def dep_inject!(derivation_rule_finder_cls)
+            @_DerivationRuleFinder = derivation_rule_finder_cls
+        end
+
+        # Returns the value for the given name by deriving from an Element in the Entry.
+        #
+        # Returns nil if no such value can be derived.
+        def fetch(entry, element_name)
+            # Find the derivation rule that will get us the element we want
+            rule = @_DerivationRuleFinder.find(element_name)
+            return nil if rule.nil?
+            
+            # Get the value of the element from which we're deriving
+            source_element_name = rule.source_name
+            source_element = entry.captured_elements[source_element_name]
+            return nil if source_element.nil?
+
+            # Do the derivation
+            rule.derive(element_name, source_element.value)
+        end
+    end
+end

data/lib/entry.rb

@@ -1,90 +1,100 @@
+require 'element'
+require 'progress'
+
 class ApacheCrunch
     # A parsed entry from the log.
     #
     # Acts like a hash, in that you get at the log elements (e.g. "url_path", "remote_host") by
     # as entry[name].
     class Entry
-        def initialize(derivation_map)
-            @_derivation_map = derivation_map
-            @_attributes = {}
-        end
+        attr_accessor :captured_elements
 
-        def []=(name, value)
-            @_attributes[name] = value
-        end
+        def initialize
+            @captured_elements = {}
+            @_value_fetcher = nil
 
-        def [](name)
-            return @_attributes[name] if @_attributes.key?(name)
-
-            derived_from_cls = @_derivation_map[name]
-            return nil if derived_from_cls.nil?
+            @_ElementValueFetcher = ElementValueFetcher
+        end
 
-            derived_from_cls.derive(name, @_attributes[derived_from_cls.name])
+        def dep_inject!(element_value_fetcher_cls)
+            @_ElementValueFetcher = element_value-fetcher_cls
         end
 
-        def merge!(hsh)
-            @_attributes.merge!(hsh)
+        def fetch(name)
+            @_value_fetcher = @_ElementValueFetcher.new if @_value_fetcher.nil?
+            @_value_fetcher.fetch(self, name)
         end
     end
 
 
     # Makes Entry instances based on log file text
     class EntryParser
-        # Initializes the instance given a LogFormat instance
-        def initialize(log_format, progress_meter)
-            @log_format = log_format
-            @progress_meter = progress_meter
+        # Initializes the instance given a ProgressMeter instance
+        def initialize
+            @_Entry = Entry
+            @_Element = Element
+            
+            @_progress_meter = NullProgressMeter.new
+            @_regex = nil
+        end
 
-            @_elements = log_format.elements
-            @_derivation_map = log_format.derivation_map
+        # Handles dependency injection
+        def dep_inject!(entry_cls, element_cls)
+            @_Entry = entry_cls
+            @_Element = element_cls
         end
 
-        # Returns a log line hash built from a line of text, or nil if the line was malformatted
+        # Applies the given ProgressMeter to the parser so that it will output progress.
         #
-        # The keys of the hash are names of LogFormatElements (e.g. "remote_host", "reqheader_referer")
-        def from_text(log_text)
-            match = (log_text =~ @log_format.regex)
+        # The meter's output_progress method will get called every time we finish parsing
+        # a log entry.
+        def add_progress_meter!(meter)
+            @_progress_meter = meter
+        end
+
+        # Returns an Entry instance built from a line of text, or nil if the line was malformatted
+        def parse(format, log_text)
+            @_regex = _build_regex(format) if @_regex.nil?
+
+            match = (log_text =~ @_regex)
             if match.nil?
-                warn "Log line did not match expected format: #{log_text}"
+                warn "Log line did not match expected format: #{log_text.rstrip}"
                 return nil
             end
-            
-            # Make a hash mapping all parsed elements to their values in the entry
+
             match_groups = Regexp.last_match.to_a
             match_groups.shift # First value is the whole matched string, which we do not want
-            element_values = Hash[*@_elements.zip(match_groups).flatten]
-
-            # Start building the return value
-            entry = Entry.new(@_derivation_map)
-            entry[:text] = log_text
-            # Insert all the elements specified in the LogFormat
-            entry.merge!(_elements_to_hash(element_values))
 
-            @progress_meter.output_progress(entry)
-            entry
-        end
-
-        # Returns a hash of "element name" => value pairs based on a hash of element => value pairs.
-        def _elements_to_hash(element_values)
-            hsh = {}
-            element_values.each_pair do |element, value|
-                hsh[element.name] = value
+            entry = @_Entry.new
+            format.captured_tokens.each_with_index do |tok,i|
+                 element = Element.new
+                 element.populate!(tok, match_groups[i])
+                 entry.captured_elements[tok.name] = element
             end
 
-            hsh
+            # Add the full text of the log entry to the Entry instance as well.
+            text_element = Element.new
+            text_element.populate!(StringToken.new, log_text)
+            entry.captured_elements[:text] = text_element
+
+            @_progress_meter.output_progress(entry)
+            entry
         end
 
-        # Returns hash of derived "element name" => value pairs from a hash of element => value pairs.
-        #
-        # That is, we go through the elements passed and if any offers derived elements, we include
-        # those in the return value.
-        def _derived_elements(element_values)
-            hsh = {}
-            element_values.each_pair do |element, value|
-                hsh.merge!(element.derived_values(value))
+        def _build_regex(format)
+            r = "^"
+            format.tokens.each do |tok|
+                # We only care to remember the captured LogFormatElements.  No need to put
+                # parentheses around StringElements that aren't interpolated.
+                if tok.captured?
+                    r += "(" + tok.regex + ")"
+                else
+                    r += tok.regex
+                end
             end
+            r += "$"
 
-            hsh
+            Regexp.compile(r)
         end
     end
 end

data/lib/format.rb

@@ -1,77 +1,38 @@
+require 'format_token'
+
 class ApacheCrunch
     # Represents a particular Apache log format
     class Format
-        attr_accessor :format_def, :tokens
+        attr_accessor :tokens
 
         def initialize
             @tokens = []
-            @_regex = nil
-        end
-
-        # Appends a given token (a LogFormatElement or LogFormatString) to the tokens list
-        def append(token)
-            @tokens << token
-        end
-
-        # Returns a compiled regex to match a log line in this format
-        #
-        # Each group matched will correspond to an element in the log format.
-        def regex
-            return @_regex unless @_regex.nil?
-
-            r = "^"
-            @tokens.each do |tok|
-                # We only care to remember the LogFormatElements.  No need to put parentheses
-                # around LogFormatString shit.
-                if tok.respond_to?(:name)
-                    r += "(" + tok.regex + ")"
-                else
-                    r += tok.regex
-                end
-            end
-            r += "$"
-
-            @_regex = Regexp.compile(r)
-            @_regex
         end
 
-        # Returns the list of LogFormatElements, in order, of the interpolated things in the format.
-        #
-        # For example, if the log format definition were "%h %u %{Referer}i", this would return the
-        # LogFormatElement instances for "%h", "%u", and "%{Referer}i".
-        def elements
+        def captured_tokens
             @tokens.find_all do |tok|
-                tok.respond_to?(:name)
+                tok.captured?
             end
         end
-
-        # Returns hash mapping names of elements to the element class from which they can be derived.
-        def derivation_map
-            hsh = {}
-            elements.each do |tok|
-                tok.derived_elements.each do |derived_element|
-                    hsh[derived_element.name] = tok.class
-                end
-            end
-
-            hsh
-        end
     end
 
     # Parses a log format definition
     class FormatParser
         # Initializes the FormatParser
         #
-        # Takes a FormatElementFactory instance, and you can inject a replacement for the
-        # LogFormatString class.
-        def initialize(format_element_factory, format_string_cls=LogFormatString)
-            @_element_factory = format_element_factory
-            @_format_string_cls = format_string_cls
+        # Takes a FormatElementFactory instance.
+        def initialize
+            @_FormatTokenFactory = FormatTokenFactory
+        end
+
+        # Handles dependency injection
+        def dep_inject!(format_token_factory_cls)
+            @_FormatTokenFactory = format_token_factory_cls
         end
 
         # Parses the given format_def (e.g. "%h %u %s #{Referer}i") and returns a list of tokens.
         #
-        # These tokens are all instances of LogFormatString or LogFormatElement.
+        # These tokens are all instances of a LogFormatElement subclass.
         def parse_def(format_def)
             s = format_def
             tokens = []
@@ -84,27 +45,27 @@ class ApacheCrunch
             tokens
         end
 
-        # Finds the first token (a LogFormatElement or LogFormatString) in a format definition
+        # Finds the first token in a format definition
         #
         # Returns a list containing the token and the new format definition (with the characters
         # that correspond to the token removed)
         def _shift_token(format_def)
             if format_def =~ /^%%(.*)/
                 # Literal "%"
-                return [@_format_string_cls.new("%%"), $1]
+                return [@_FormatTokenFactory.from_abbrev("%%"), $1]
             elsif format_def =~ /^(%[A-Za-z])(.*)/
                 # Simple element (e.g. "%h", "%u")
-                return [@_element_factory.from_abbrev($1), $2]
+                return [@_FormatTokenFactory.from_abbrev($1), $2]
             elsif format_def =~ /^%[<>]([A-Za-z])(.*)/
                 # No idea how to handle mod_log_config's "which request" system yet, so we
                 # ignore it.
-                return [@_element_factory.from_abbrev("%" + $1), $2]
+                return [@_FormatTokenFactory.from_abbrev("%" + $1), $2]
             elsif format_def =~ /^(%\{.+?\}[Ceinor])(.*)/
                 # "Contents of" element (e.g. "%{Accept}i")
-                return [@_element_factory.from_abbrev($1), $2]
+                return [@_FormatTokenFactory.from_abbrev($1), $2]
             elsif format_def =~ /^(.+?)(%.*|$)/
                 # Bare string up until the next %, or up until the end of the format definition
-                return [@_format_string_cls.new($1), $2]
+                return [@_FormatTokenFactory.from_abbrev($1), $2]
             end
         end
     end
@@ -115,11 +76,8 @@ class ApacheCrunch
         # Constructs and returns a Format instance based on the given Apache log format string
         def self.from_format_def(format_def)
             logformat = Format.new
-            logformat.format_def = format_def
-
-            element_factory = LogFormatElementFactory.new
 
-            format_parser = FormatParser.new(element_factory)
+            format_parser = FormatParser.new
             logformat.tokens = format_parser.parse_def(format_def)
 
             logformat