apachecrunch 0.3 → 0.4

data/bin/apachecrunch

@@ -60,11 +60,11 @@ end
 
 options = parse_args
 
-format_string = FormatStringFinder.new.find(options[:format])
+format_def = ApacheCrunch::FormatDefinitionFinder.new.find(options[:format])
 progress_meter = ProgressMeterFactory.from_options(options)
-log_parser = LogParserFactory.log_parser(
-                        format_string=format_string,
-                        path=options[:logfile],
-                        progress_meter=progress_meter)
+log_parser = ApacheCrunch::LogParserFactory.log_parser(
+                        format_def,
+                        options[:logfile],
+                        progress_meter)
 proc_env = ProcedureEnvironment.new(log_parser)
 proc_env.eval_procedure(open(options[:procedure]).read())

data/lib/apachecrunch.rb

@@ -1,320 +1,22 @@
 require "date"
 require "tempfile"
 
+require 'config'
+require 'entry'
+require 'format'
+require 'log_parser'
 require 'log_element'
 
-
-# A parsed entry from the log.
-#
-# Acts like a hash, in that you get at the log elements (e.g. "url_path", "remote_host") by
-# as entry[name].
-class LogEntry
-    def initialize(derivation_map)
-        @_derivation_map = derivation_map
-        @_attributes = {}
-    end
-
-    def []=(name, value)
-        @_attributes[name] = value
-    end
-
-    def [](name)
-        return @_attributes[name] if @_attributes.key?(name)
-
-        derived_from_cls = @_derivation_map[name]
-        return nil if derived_from_cls.nil?
-
-        derived_from_cls.derive(name, @_attributes[derived_from_cls.name])
-    end
-
-    def merge!(hsh)
-        @_attributes.merge!(hsh)
-    end
-end
-
-
-# A bare string in a log format
-#
-# Exposes 'regex' for consistency with LogFormatElement, but there shouldn't be anything other
-# than one-to-one character matching in there.
-class LogFormatString
-    attr_accessor :regex
-
-    def initialize(regex)
-        @regex = regex
-    end
-end
-
-
-# Represents a particular Apache log format
-class LogFormat
-    attr_accessor :format_string, :tokens
-
-    def initialize
-        @tokens = []
-        @_regex = nil
-    end
-
-    # Appends a given token (a LogFormatElement or LogFormatString) to the tokens list
-    def append(token)
-        @tokens << token
-    end
-
-    # Returns a compiled regex to match a log line in this format
-    def regex
-        return @_regex unless @_regex.nil?
-
-        r = "^"
-        @tokens.each do |tok|
-            # We only care to remember the LogFormatElements.  No need to put parentheses
-            # around LogFormatString shit.
-            if tok.respond_to?(:name)
-                r += "(" + tok.regex + ")"
-            else
-                r += tok.regex
-            end
-        end
-        r += "$"
-
-        @_regex = Regexp.compile(r)
-        @_regex
-    end
-
-    # Returns the list of LogFormatElements, in order, of the interpolated things in the format.
+class ApacheCrunch
+    # A bare string in a log format
     #
-    # For example, if the log format string were "%h %u %{Referer}i", this would return the
-    # LogFormatElement instances for "%h", "%u", and "%{Referer}i".
-    def elements
-        @tokens.find_all do |tok|
-            tok.respond_to?(:name)
-        end
-    end
-
-    # Returns hash mapping names of elements to the element class from which they can be derived.
-    def derivation_map
-        hsh = {}
-        elements.each do |tok|
-            tok.derived_elements.each do |derived_element|
-                hsh[derived_element.name] = tok.class
-            end
-        end
-
-        hsh
-    end
-end
-
-
-# Turns a string specifying an Apache log format into a LogFormat instance
-class LogFormatFactory
-    def initialize
-        @element_factory = LogFormatElementFactory.new
-    end
-
-    # Constructs and returns a LogFormat instance based on the given Apache log format string
-    def from_format_string(f_string)
-        logformat = LogFormat.new
-        logformat.format_string = f_string
+    # Exposes 'regex' for consistency with LogFormatElement, but there shouldn't be anything other
+    # than one-to-one character matching in there.
+    class LogFormatString
+        attr_accessor :regex
 
-        until f_string.empty?
-            token, f_string = _shift_token(f_string)
-            logformat.append(token)
+        def initialize(regex)
+            @regex = regex
         end
-
-        logformat
-    end
-
-    # Finds the first token (a LogFormatElement or LogFormatString) in a format string
-    #
-    # Returns a list containing the token and the new format string (with the characters that
-    # correspond to the token removed)
-    def _shift_token(f_string)
-        if f_string =~ /^%%(.*)/
-            # Literal "%"
-            return [LogFormatString.new("%%"), $1]
-        elsif f_string =~ /^(%[A-Za-z])(.*)/
-            # Simple element (e.g. "%h", "%u")
-            return [@element_factory.from_abbrev($1), $2]
-        elsif f_string =~ /^%[<>]([A-Za-z])(.*)/
-            # No idea how to handle mod_log_config's "which request" system yet, so we
-            # ignore it.
-            return [@element_factory.from_abbrev("%" + $1), $2]
-        elsif f_string =~ /^(%\{.+?\}[Ceinor])(.*)/
-            # "Contents of" element (e.g. "%{Accept}i")
-            return [@element_factory.from_abbrev($1), $2]
-        elsif f_string =~ /^(.+?)(%.*|$)/
-            # Bare string up until the next %, or up until the end of the format string
-            return [LogFormatString.new($1), $2]
-        end
-    end
-end
-
-
-# Makes log line hashes based on log file text
-class LogLineParser
-    # Initializes the instance given a LogFormat instance
-    def initialize(log_format, progress_meter)
-        @log_format = log_format
-        @progress_meter = progress_meter
-
-        @_elements = log_format.elements
-        @_derivation_map = log_format.derivation_map
-    end
-
-    # Returns a log line hash built from a line of text, or nil if the line was malformatted
-    #
-    # The keys of the hash are names of LogFormatElements (e.g. "remote_host", "reqheader_referer")
-    def from_text(log_text)
-        match = (log_text =~ @log_format.regex)
-        if match.nil?
-            warn "Log line did not match expected format: #{log_text}"
-            return nil
-        end
-        
-        # Make a hash mapping all parsed elements to their values in the entry
-        match_groups = Regexp.last_match.to_a
-        match_groups.shift # First value is the whole matched string, which we do not want
-        element_values = Hash[*@_elements.zip(match_groups).flatten]
-
-        # Start building the return value
-        entry = LogEntry.new(@_derivation_map)
-        entry[:text] = log_text
-        # Insert all the elements specified in the LogFormat
-        entry.merge!(_elements_to_hash(element_values))
-
-        @progress_meter.output_progress(entry)
-        entry
-    end
-
-    # Returns a hash of "element name" => value pairs based on a hash of element => value pairs.
-    def _elements_to_hash(element_values)
-        hsh = {}
-        element_values.each_pair do |element, value|
-            hsh[element.name] = value
-        end
-
-        hsh
-    end
-
-    # Returns hash of derived "element name" => value pairs from a hash of element => value pairs.
-    #
-    # That is, we go through the elements passed and if any offers derived elements, we include
-    # those in the return value.
-    def _derived_elements(element_values)
-        hsh = {}
-        element_values.each_pair do |element, value|
-            hsh.merge!(element.derived_values(value))
-        end
-
-        hsh
-    end
-end
-
-
-# Parses a log file given a path and a LogFormat instance
-class LogParser
-    # Initializes the parser with the path to a log file and a LogLineParser.
-    def initialize(path, ll_parser)
-        @path = path
-        @ll_parser = ll_parser
-
-        @_file = nil
-    end
-
-    # Returns the next entry in the log file as a hash, or nil if we've reached EOF.
-    #
-    # The keys of the hash are names of LogFormatElements (e.g. "remote_host", "reqheader_referer")
-    def next_entry
-        @_file = open(@path) if @_file.nil?
-
-        while line_text = @_file.gets
-            return nil if line_text.nil?
-            logline = @ll_parser.from_text(line_text)
-
-            # The LogLineFactory returns nil and writes a warning if the line text doesn't
-            # match our expected format.
-            next if logline.nil?
-
-            return logline
-        end
-    end
-
-    # Resets the LogParser's filehandle so we can start over.
-    def reset
-        @_file = nil
-    end
-
-    # Makes the LogParser close its current log file and start parsing a new one instead
-    #
-    # `new_target` is a writable file object that the parser should start parsing, and if
-    # in_place is true, we actually replace the contents of the current target with those
-    # of the new target.
-    def replace_target(new_target, in_place)
-        new_target.close
-
-        if in_place
-            old_path = @_file.path
-            File.rename(new_target.path, old_path)
-        else
-            @path = new_target.path
-        end
-
-        @_file = nil
-    end
-end
-
-# Makes a LogParser given the parameters we want to work with.
-#
-# This is the class that most external code should instatiate to begin using this library.
-class LogParserFactory
-    # Returns a new LogParser instance for the given log file, which should have the given Apache
-    # log format.
-    def self.log_parser(format_string, path, progress_meter)
-        # First we generate a LogFormat instance based on the format string we were given
-        format_factory = LogFormatFactory.new
-        log_format = format_factory.from_format_string(format_string)
-
-        # Now we generate a line parser
-        log_line_parser = LogLineParser.new(log_format, progress_meter)
-
-        # And now we can instantiate and return a LogParser
-        return LogParser.new(path, log_line_parser)
-    end
-end
-
-
-# Finds a named log format string in the configuration file(s)
-class FormatStringFinder
-    @@FILE_NAME = "log_formats.rb"
-    @@DEFAULT_FORMATS = {
-        :ncsa => %q!%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"!,
-        :ubuntu => %q!%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"!
-    }
-
-    # Finds the given format string in the configuration file(s)
-    #
-    # If none exists, returns nil.
-    def find(format_name)
-        name_as_symbol = format_name.to_sym
-
-        formats = @@DEFAULT_FORMATS.clone
-        _search_path.each do |dir|
-            config_path = File.join(dir, @@FILE_NAME)
-            if File.readable?(config_path)
-                config_file = open(File.join(dir, @@FILE_NAME))
-                eval config_file.read
-            end
-
-            if formats.key?(format_name.to_sym)
-                return formats[format_name.to_sym].gsub(/\\"/, '"')
-            end
-        end
-
-        raise "Failed to find the format '#{format_name}' in the search path: #{_search_path.inspect}"
-    end
-
-    def _search_path
-        [".", "./etc",
-         File.join(ENV["HOME"], ".apachecrunch"),
-         "/etc/apachecrunch"]
     end
 end

data/lib/config.rb

@@ -0,0 +1,44 @@
+class ApacheCrunch
+    # Finds a named log format string in the configuration file(s)
+    class FormatDefinitionFinder
+        @@FILE_NAME = "log_formats.rb"
+        @@DEFAULT_FORMATS = {
+            :ncsa => %q!%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"!,
+            :ubuntu => %q!%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"!
+        }
+
+        # Initializes the FormatStringFinder.
+        def initialize(file_cls=File, env=ENV)
+            @_file_cls=file_cls
+            @_env=env
+        end
+
+        # Finds the given format string in the configuration file(s)
+        #
+        # If none exists, returns nil.
+        def find(format_name)
+            name_as_symbol = format_name.to_sym
+
+            formats = @@DEFAULT_FORMATS.clone
+            _search_path.each do |dir|
+                config_path = @_file_cls.join(dir, @@FILE_NAME)
+                if @_file_cls.readable?(config_path)
+                    config_file = @_file_cls.open(@_file_cls.join(dir, @@FILE_NAME))
+                    eval config_file.read
+                end
+
+                if formats.key?(format_name.to_sym)
+                    return formats[format_name.to_sym].gsub(/\\"/, '"')
+                end
+            end
+
+            raise "Failed to find the format '#{format_name}' in the search path: #{_search_path.inspect}"
+        end
+
+        def _search_path
+            [".", "./etc",
+             @_file_cls.join(@_env["HOME"], ".apachecrunch"),
+             "/etc/apachecrunch"]
+        end
+    end
+end

data/lib/entry.rb

@@ -0,0 +1,90 @@
+class ApacheCrunch
+    # A parsed entry from the log.
+    #
+    # Acts like a hash, in that you get at the log elements (e.g. "url_path", "remote_host") by
+    # as entry[name].
+    class Entry
+        def initialize(derivation_map)
+            @_derivation_map = derivation_map
+            @_attributes = {}
+        end
+
+        def []=(name, value)
+            @_attributes[name] = value
+        end
+
+        def [](name)
+            return @_attributes[name] if @_attributes.key?(name)
+
+            derived_from_cls = @_derivation_map[name]
+            return nil if derived_from_cls.nil?
+
+            derived_from_cls.derive(name, @_attributes[derived_from_cls.name])
+        end
+
+        def merge!(hsh)
+            @_attributes.merge!(hsh)
+        end
+    end
+
+
+    # Makes Entry instances based on log file text
+    class EntryParser
+        # Initializes the instance given a LogFormat instance
+        def initialize(log_format, progress_meter)
+            @log_format = log_format
+            @progress_meter = progress_meter
+
+            @_elements = log_format.elements
+            @_derivation_map = log_format.derivation_map
+        end
+
+        # Returns a log line hash built from a line of text, or nil if the line was malformatted
+        #
+        # The keys of the hash are names of LogFormatElements (e.g. "remote_host", "reqheader_referer")
+        def from_text(log_text)
+            match = (log_text =~ @log_format.regex)
+            if match.nil?
+                warn "Log line did not match expected format: #{log_text}"
+                return nil
+            end
+            
+            # Make a hash mapping all parsed elements to their values in the entry
+            match_groups = Regexp.last_match.to_a
+            match_groups.shift # First value is the whole matched string, which we do not want
+            element_values = Hash[*@_elements.zip(match_groups).flatten]
+
+            # Start building the return value
+            entry = Entry.new(@_derivation_map)
+            entry[:text] = log_text
+            # Insert all the elements specified in the LogFormat
+            entry.merge!(_elements_to_hash(element_values))
+
+            @progress_meter.output_progress(entry)
+            entry
+        end
+
+        # Returns a hash of "element name" => value pairs based on a hash of element => value pairs.
+        def _elements_to_hash(element_values)
+            hsh = {}
+            element_values.each_pair do |element, value|
+                hsh[element.name] = value
+            end
+
+            hsh
+        end
+
+        # Returns hash of derived "element name" => value pairs from a hash of element => value pairs.
+        #
+        # That is, we go through the elements passed and if any offers derived elements, we include
+        # those in the return value.
+        def _derived_elements(element_values)
+            hsh = {}
+            element_values.each_pair do |element, value|
+                hsh.merge!(element.derived_values(value))
+            end
+
+            hsh
+        end
+    end
+end

data/lib/format.rb

@@ -0,0 +1,128 @@
+class ApacheCrunch
+    # Represents a particular Apache log format
+    class Format
+        attr_accessor :format_def, :tokens
+
+        def initialize
+            @tokens = []
+            @_regex = nil
+        end
+
+        # Appends a given token (a LogFormatElement or LogFormatString) to the tokens list
+        def append(token)
+            @tokens << token
+        end
+
+        # Returns a compiled regex to match a log line in this format
+        #
+        # Each group matched will correspond to an element in the log format.
+        def regex
+            return @_regex unless @_regex.nil?
+
+            r = "^"
+            @tokens.each do |tok|
+                # We only care to remember the LogFormatElements.  No need to put parentheses
+                # around LogFormatString shit.
+                if tok.respond_to?(:name)
+                    r += "(" + tok.regex + ")"
+                else
+                    r += tok.regex
+                end
+            end
+            r += "$"
+
+            @_regex = Regexp.compile(r)
+            @_regex
+        end
+
+        # Returns the list of LogFormatElements, in order, of the interpolated things in the format.
+        #
+        # For example, if the log format definition were "%h %u %{Referer}i", this would return the
+        # LogFormatElement instances for "%h", "%u", and "%{Referer}i".
+        def elements
+            @tokens.find_all do |tok|
+                tok.respond_to?(:name)
+            end
+        end
+
+        # Returns hash mapping names of elements to the element class from which they can be derived.
+        def derivation_map
+            hsh = {}
+            elements.each do |tok|
+                tok.derived_elements.each do |derived_element|
+                    hsh[derived_element.name] = tok.class
+                end
+            end
+
+            hsh
+        end
+    end
+
+    # Parses a log format definition
+    class FormatParser
+        # Initializes the FormatParser
+        #
+        # Takes a FormatElementFactory instance, and you can inject a replacement for the
+        # LogFormatString class.
+        def initialize(format_element_factory, format_string_cls=LogFormatString)
+            @_element_factory = format_element_factory
+            @_format_string_cls = format_string_cls
+        end
+
+        # Parses the given format_def (e.g. "%h %u %s #{Referer}i") and returns a list of tokens.
+        #
+        # These tokens are all instances of LogFormatString or LogFormatElement.
+        def parse_def(format_def)
+            s = format_def
+            tokens = []
+            
+            until s.empty?
+                token, s = _shift_token(s)
+                tokens << token
+            end
+
+            tokens
+        end
+
+        # Finds the first token (a LogFormatElement or LogFormatString) in a format definition
+        #
+        # Returns a list containing the token and the new format definition (with the characters
+        # that correspond to the token removed)
+        def _shift_token(format_def)
+            if format_def =~ /^%%(.*)/
+                # Literal "%"
+                return [@_format_string_cls.new("%%"), $1]
+            elsif format_def =~ /^(%[A-Za-z])(.*)/
+                # Simple element (e.g. "%h", "%u")
+                return [@_element_factory.from_abbrev($1), $2]
+            elsif format_def =~ /^%[<>]([A-Za-z])(.*)/
+                # No idea how to handle mod_log_config's "which request" system yet, so we
+                # ignore it.
+                return [@_element_factory.from_abbrev("%" + $1), $2]
+            elsif format_def =~ /^(%\{.+?\}[Ceinor])(.*)/
+                # "Contents of" element (e.g. "%{Accept}i")
+                return [@_element_factory.from_abbrev($1), $2]
+            elsif format_def =~ /^(.+?)(%.*|$)/
+                # Bare string up until the next %, or up until the end of the format definition
+                return [@_format_string_cls.new($1), $2]
+            end
+        end
+    end
+
+
+    # Turns a string specifying an Apache log format into a Format instance
+    class FormatFactory
+        # Constructs and returns a Format instance based on the given Apache log format string
+        def self.from_format_def(format_def)
+            logformat = Format.new
+            logformat.format_def = format_def
+
+            element_factory = LogFormatElementFactory.new
+
+            format_parser = FormatParser.new(element_factory)
+            logformat.tokens = format_parser.parse_def(format_def)
+
+            logformat
+        end
+    end
+end

data/lib/log_parser.rb

@@ -0,0 +1,74 @@
+class ApacheCrunch
+    # Parses a log file given a path and a Format instance
+    class LogParser
+        # Initializes the parser with the path to a log file and a EntryParser.
+        def initialize(path, entry_parser, file_cls=File)
+            @path = path
+            @entry_parser = entry_parser
+
+            @_file_cls = file_cls
+            @_file = nil
+        end
+
+        # Returns the next entry in the log file as a hash, or nil if we've reached EOF.
+        #
+        # The keys of the hash are names of LogFormatElements (e.g. :remote_host, 
+        # :reqheader_referer)
+        def next_entry
+            @_file = @_file_cls.open(@path) if @_file.nil?
+
+            while line_text = @_file.gets
+                return nil if line_text.nil?
+                logline = @entry_parser.from_text(line_text)
+
+                # The EntryParser returns nil and writes a warning if the line text doesn't
+                # match our expected format.
+                next if logline.nil?
+
+                return logline
+            end
+        end
+
+        # Resets the LogParser's filehandle so we can start over.
+        def reset
+            @_file = nil
+        end
+
+        # Makes the LogParser close its current log file and start parsing a new one instead
+        #
+        # `new_target` is a writable file object that the parser should start parsing, and if
+        # in_place is true, we actually replace the contents of the current target with those
+        # of the new target.
+        def replace_target(new_target, in_place)
+            new_target.close
+
+            if in_place
+                old_path = @_file.path
+                @_file_cls.rename(new_target.path, old_path)
+            else
+                @path = new_target.path
+            end
+
+            @_file = nil
+        end
+    end
+
+
+    # Makes a LogParser given the parameters we want to work with.
+    #
+    # This is the class that most external code should instatiate to begin using this library.
+    class LogParserFactory
+        # Returns a new LogParser instance for the given log file, which should have the given
+        # Apache log format.
+        def self.log_parser(format_def, path, progress_meter)
+            # First we generate a Format instance based on the format definition we were given
+            log_format = FormatFactory.from_format_def(format_def)
+
+            # Now we generate a line parser
+            log_line_parser = EntryParser.new(log_format, progress_meter)
+
+            # And now we can instantiate and return a LogParser
+            return LogParser.new(path, log_line_parser)
+        end
+    end
+end

data/lib/progress.rb

@@ -39,7 +39,7 @@ class TimeProgressMeter < ProgressMeter
     def output_progress(entry)
         @_entry_count += 1
         if @_entry_count % @_period == 0
-            puts "Processed through %s" % [entry["time"]]
+            puts "Processed through %s" % [entry[:time]]
         end
     end
 end

data/test/runner.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+require 'test/unit'
+
+$: << ".."
+$: << "./lib"
+require 'apachecrunch'
+
+class AllTests
+    def self.run
+        Dir.glob("test/test_*.rb").each do |test_file|
+            require test_file
+        end
+    end
+end
+
+AllTests.run

data/test/stub.rb

@@ -0,0 +1,56 @@
+class StubFormatElement < LogFormatElement
+    @abbrev = "%z"
+    @name = :stub
+    @regex = %q!.*!
+end
+
+class StubAlphanumericFormatElement < LogFormatElement
+    @abbrev = "%Z"
+    @name = :alnum
+    @regex = %q![A-Za-z0-9]+!
+end
+
+class StubNumericFormatElement < LogFormatElement
+    @abbrev = "%y"
+    @name = :num
+    @regex = %q!\d+!
+end
+
+class StubFormatString
+    attr_accessor :regex
+    def initialize(regex)
+        @regex = regex
+    end
+end
+
+class StubDerivedElement < LogFormatElement
+    @abbrev = ""
+    @name = :derived
+    @regex = %q!.*!
+end
+
+class StubDerivationSourceElement < LogFormatElement
+    @name = :derivation_source
+
+    def derived_elements
+        [StubDerivedElement]
+    end
+
+    def self.derive(name, our_own_value)
+        if name == :derived
+            return "derived from #{our_own_value}"
+        end
+
+        nil
+    end
+end
+
+class StubLogFormatElementFactory
+    def from_abbrev(abbrev)
+        if abbrev =~ /^%/
+            return StubFormatElement.new
+        else
+            return StubFormatString.new
+        end
+    end
+end

data/test/test_entry.rb

@@ -0,0 +1,28 @@
+require 'test/stub'
+
+class TestEntry < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::Entry.new({:derived => StubDerivationSourceElement})
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    # Tests direct assignment of an element.
+    def test_assign
+        @inst[:bar] = "test_value"
+        assert_equal(@inst[:bar], "test_value")
+    end
+
+    # Tests derivation of one element from another.
+    def test_derive
+        @inst[:derivation_source] = "source text"
+        assert_equal(@inst[:derived], "derived from source text")
+    end
+
+    # Tests access to an absent element.
+    def test_access_absent
+        assert_nil(@inst[:nonexistent])
+    end
+end

data/test/test_format.rb

@@ -0,0 +1,63 @@
+require 'test/stub'
+
+class TestFormat < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::Format.new
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    # Tests appending an element to the format
+    def test_append
+        format_element = StubFormatElement.new
+        @inst.append(format_element)
+        assert_same(@inst.tokens[-1], format_element)
+    end
+
+    # Tests regex compilation for a simple format
+    def test_regex_simple
+        @inst.append(StubAlphanumericFormatElement.new)
+        "abc123\n" =~ @inst.regex
+        assert_equal($1, "abc123")
+
+        "!abc123\n" =~ @inst.regex
+        assert_nil($1)
+    end
+
+    # Tests regex compilation for a more complex format
+    def test_regex_complex
+        @inst.append(StubNumericFormatElement.new)
+        @inst.append(StubFormatString.new(' \(some stuff\) '))
+        @inst.append(StubAlphanumericFormatElement.new)
+
+        "54321 (some stuff) alphaNumericStuff" =~ @inst.regex
+        assert_equal([Regexp.last_match(1), Regexp.last_match(2)],
+                      ["54321", "alphaNumericStuff"])
+
+        "54321 (doesn't match) alphaNumericStuff" =~ @inst.regex
+        assert_equal([Regexp.last_match(1), Regexp.last_match(2)],
+                      [nil, nil])
+    end
+
+    # Tests the list of matchable elements
+    def test_elements
+        num_element = StubNumericFormatElement.new
+        alnum_element = StubAlphanumericFormatElement.new
+        @inst.append(num_element)
+        @inst.append(StubFormatString.new(' \(some stuff\) '))
+        @inst.append(alnum_element)
+
+        assert_equal(@inst.elements, [num_element, alnum_element])
+    end
+
+    # Tests the derivation map
+    def test_derivation_map
+        @inst.append(StubNumericFormatElement.new)
+        @inst.append(StubFormatString.new(' \(some stuff\) '))
+        @inst.append(StubDerivationSourceElement.new)
+
+        assert(@inst.derivation_map, {:derived => StubDerivationSourceElement})
+    end
+end

data/test/test_format_parser.rb

@@ -0,0 +1,26 @@
+require 'test/stub'
+
+class TestFormatParser < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::FormatParser.new(StubLogFormatElementFactory.new,
+                                               StubFormatString)
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    def test_parse_simple
+        tokens = @inst.parse_def("%Z %z")
+        [StubFormatElement, StubFormatString, StubFormatElement].each_with_index do |c,i|
+            assert_instance_of(c, tokens[i])
+        end
+    end
+
+    def test_parse_complex
+        tokens = @inst.parse_def("%{Foo-Bar}i %{baz:\d+}r")
+        [StubFormatElement, StubFormatString, StubFormatElement].each_with_index do |c,i|
+            assert_instance_of(c, tokens[i])
+        end
+    end
+end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: apachecrunch
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 3
   prerelease: 
   segments: 
   - 0
-  - 3
-  version: "0.3"
+  - 4
+  version: "0.4"
 platform: ruby
 authors: 
 - Dan Slimmon
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-10 00:00:00 -04:00
+date: 2011-07-12 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
@@ -31,11 +31,20 @@ extra_rdoc_files: []
 
 files: 
 - lib/apachecrunch.rb
+- lib/config.rb
+- lib/entry.rb
+- lib/format.rb
 - lib/log_element.rb
+- lib/log_parser.rb
 - lib/procedure_dsl.rb
 - lib/progress.rb
 - bin/apachecrunch
 - LICENSE
+- test/runner.rb
+- test/stub.rb
+- test/test_entry.rb
+- test/test_format.rb
+- test/test_format_parser.rb
 has_rdoc: true
 homepage: https://github.com/danslimmon/apachecrunch/
 licenses: