apachecrunch 0.4 → 0.5

data/test/test_log_parser.rb

@@ -0,0 +1,88 @@
+require 'test/stub'
+require 'test/mock'
+
+class TestLogParser < Test::Unit::TestCase
+    def setup
+        @entry_parser = StubEntryParser.new
+        @inst = ApacheCrunch::LogParser.new(@entry_parser)
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    # Tests setting the log file to a given file
+    def test_set_file
+        first_log_file = MockFile.new
+        assert_nothing_thrown("#{@inst.class}#set_file! threw an exception") do
+            @inst.set_file!(first_log_file)
+        end
+
+        second_log_file = MockFile.new
+        assert_nothing_thrown("#{@inst.class}#set_file! threw an exception") do
+            @inst.set_file!(second_log_file)
+        end
+        assert_equal(1, first_log_file.close_count,
+                     "#{@inst.class}#set_file! didn't close old file")
+    end
+
+    # Tests replacing the log file with another file
+    def test_replace_file
+        mock_file_class = MockFileClass.new
+        @inst.dep_inject!(mock_file_class)
+
+        first_log_file = MockFile.new
+        first_log_file.path = "/first/path"
+        @inst.set_file!(first_log_file)
+
+        second_log_file = MockFile.new
+        second_log_file.path = "/second/path"
+        assert_nothing_thrown("#{@inst.class}#replace_file! threw an exception") do
+            @inst.replace_file!(second_log_file)
+        end
+        assert_equal([["/second/path", "/first/path"]], mock_file_class.rename_calls,
+                     "#{@inst.class}#replace_file! didn't move the file into place")
+        assert_equal(1, first_log_file.close_count,
+                     "#{@inst.class}#replace_file! didn't close old file")
+    end
+
+    # Tests resetting the log file to its beginning
+    def test_reset_file
+        mock_file_class = MockFileClass.new
+        @inst.dep_inject!(mock_file_class)
+
+        log_file = MockFile.new
+        @inst.set_file!(log_file)
+
+        assert_nothing_thrown("#{@inst.class}#reset_file! threw an exception") do
+            @inst.reset_file!
+        end
+        assert_equal(1, log_file.close_count,
+                     "#{@inst.class}#reset_file! didn't close the log file")
+        assert_equal(1, mock_file_class.open_calls.length,
+                     "#{@inst.class}#reset_file! didn't reopen the log file")
+    end
+
+    # Tests retrieving the next entry in the log
+    def test_next_entry
+        @entry_parser.parse_return_values = [
+            StubEntry.new,
+            StubEntry.new,
+            nil,
+            StubEntry.new
+        ]
+
+        log_file = MockFile.new
+        log_file.lines = ["a\n", "b\n", "c\n", "d\n"]
+
+        @inst.set_file!(log_file)
+        @inst.set_format!(StubFormat.new)
+        assert_instance_of(StubEntry, @inst.next_entry,
+                           "#{@inst.class}#next_entry returned wrong type of thing")
+        assert_instance_of(StubEntry, @inst.next_entry,
+                           "#{@inst.class}#next_entry returned wrong second value")
+        assert_instance_of(StubEntry, @inst.next_entry,
+                           "#{@inst.class}#next_entry returned wrong value when it should have skipped a malformatted entry")
+        assert_nil(@inst.next_entry, "#{@inst.class}#next_entry returned non-nil at EOF")
+    end
+end

data/test/test_raw_value_fetcher.rb

@@ -0,0 +1,36 @@
+require 'test/stub'
+
+class TestRawValueFetcher < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::RawValueFetcher.new
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    # Tests a successful fetch call
+    def test_fetch
+        entry = StubEntry.new
+        alnum_element = StubElement.new
+        alnum_element.populate!(StubAlphanumericFormatToken.new, "foo123")
+        num_element = StubElement.new
+        num_element.populate!(StubNumericFormatToken.new, 54321)
+
+        entry.captured_elements = {:alnum => alnum_element, :num => num_element}
+        assert_equal("foo123", @inst.fetch(entry, StubAlphanumericFormatToken.new.name))
+        assert_equal(54321, @inst.fetch(entry, StubNumericFormatToken.new.name))
+    end
+
+    # Tests a fetch call for an element that's not there
+    def test_fetch_missing
+        entry = StubEntry.new
+        alnum_element = StubElement.new
+        alnum_element.populate!(StubAlphanumericFormatToken.new, "foo123")
+        num_element = StubElement.new
+        num_element.populate!(StubNumericFormatToken.new, 54321)
+
+        entry.captured_elements = {:alnum => alnum_element, :num => num_element}
+        assert_nil(@inst.fetch(entry, :missing_element))
+    end
+end

data/test/test_regex_token.rb

@@ -0,0 +1,17 @@
+class TestReqheaderToken < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::RegexToken.new
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    # Tests that the token figures out its name correctly from the header
+    def test_name
+        @inst.populate!("foobar", "[A-Za-z0-9]+")
+
+        assert_equal(:regex_foobar, @inst.name,
+                     "#{@inst.class} got wrong name based on regex name 'foobar'")
+    end
+end

data/test/test_req_firstline_derivation_rule.rb

@@ -0,0 +1,41 @@
+require 'test/stub'
+
+class TestReqFirstlineDerivationRule < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::ReqFirstlineDerivationRule.new
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    def test_derive
+        expected_names = @inst.target_names
+        datasets = [
+            ["GET / HTTP/1.1", {:req_method => "GET",
+                                :url_path => "/",
+                                :query_string => "",
+                                :protocol => "HTTP/1.1"}],
+            ["HEAD /?herp=derp&foo=bar HTTP/1.0", {:req_method => "HEAD",
+                                                   :url_path => "/",
+                                                   :query_string => "?herp=derp&foo=bar",
+                                                   :protocol => "HTTP/1.0"}],
+            ["POST /some/page?never=gonna HTTP/1.1", {:req_method => "POST",
+                                                      :url_path => "/some/page",
+                                                      :query_string => "?never=gonna",
+                                                      :protocol => "HTTP/1.1"}]
+        ]
+
+        expected_names = @inst.target_names
+        datasets.each do |ds|
+            firstline_value = ds[0]
+            expected_values = ds[1]
+
+            expected_names = [:req_method, :url_path, :query_string, :protocol]
+            expected_names.each do |name|
+                assert_equal(expected_values[name], @inst.derive(name, firstline_value),
+                             "#{@inst.class}#derive returned wrong value for element '#{name}'")
+            end
+        end
+    end
+end

data/test/test_reqheader_token.rb

@@ -0,0 +1,26 @@
+class TestReqheaderToken < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::ReqheaderToken.new
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    # Tests that the token figures out its name correctly from the header
+    def test_name
+        datasets = [
+            ['Host', :reqheader_host],
+            ['X-Two-Words', :reqheader_x_two_words]
+        ]
+
+        datasets.each do |ds|
+            header_name = ds[0]
+            expected_token_name = ds[1]
+
+            @inst.populate!(header_name)
+            assert_equal(expected_token_name, @inst.name,
+                         "#{@inst.class} got wrong name based on header name '#{header_name}'")
+        end
+    end
+end

data/test/test_string_token.rb

@@ -0,0 +1,27 @@
+require 'test/stub'
+
+class TestStringToken < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::StringToken.new
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    def test_regex
+        datasets = [
+            [' ', '\ '],
+            [' {has (regex [chars', '\ \{has\ \(regex\ \[chars']
+        ]
+
+        datasets.each do |ds|
+            string_value = ds[0]
+            expected_regex = ds[1]
+
+            @inst.populate!(string_value)
+            assert_equal(expected_regex, @inst.regex,
+                         "#{@inst.class}#regex returned incorrect value")
+        end
+    end
+end

data/test/test_time_derivation_rule.rb

@@ -0,0 +1,29 @@
+require 'test/stub'
+
+class TestTimeDerivationRule < Test::Unit::TestCase
+    def setup
+        @inst = ApacheCrunch::TimeDerivationRule.new
+    end
+
+    def teardown
+        @inst = nil
+    end
+
+    # Tests that the expected list of derived elements matches what actually gets derived
+    def test_derive
+        expected_names = [:year, :month, :day, :hour, :minute, :second]
+        expected_values = {
+            :year => 2011,
+            :month => 7,
+            :day => 15,
+            :hour => 9,
+            :minute => 55,
+            :second => 41
+        }
+
+        expected_names.each do |name|
+            assert_equal(expected_values[name], @inst.derive(name, "[15/Jul/2011:09:55:41 +0400]"),
+                         "#{@inst.class}#derive_all returned incorrect value for element #{name}")
+        end
+    end
+end

metadata

@@ -1,12 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: apachecrunch
 version: !ruby/object:Gem::Version 
-  hash: 3
-  prerelease: 
-  segments: 
-  - 0
-  - 4
-  version: "0.4"
+  version: "0.5"
 platform: ruby
 authors: 
 - Dan Slimmon
@@ -14,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-12 00:00:00 -04:00
+date: 2011-09-17 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
@@ -31,20 +26,36 @@ extra_rdoc_files: []
 
 files: 
 - lib/apachecrunch.rb
+- lib/cast.rb
 - lib/config.rb
+- lib/derivation.rb
+- lib/element.rb
+- lib/element_value_fetcher.rb
 - lib/entry.rb
 - lib/format.rb
-- lib/log_element.rb
+- lib/format_token.rb
+- lib/format_token_definition.rb
 - lib/log_parser.rb
 - lib/procedure_dsl.rb
 - lib/progress.rb
 - bin/apachecrunch
 - LICENSE
+- test/mock.rb
 - test/runner.rb
 - test/stub.rb
-- test/test_entry.rb
+- test/test_derived_value_fetcher.rb
+- test/test_element.rb
+- test/test_element_value_fetcher.rb
+- test/test_entry_parser.rb
 - test/test_format.rb
 - test/test_format_parser.rb
+- test/test_log_parser.rb
+- test/test_raw_value_fetcher.rb
+- test/test_regex_token.rb
+- test/test_req_firstline_derivation_rule.rb
+- test/test_reqheader_token.rb
+- test/test_string_token.rb
+- test/test_time_derivation_rule.rb
 has_rdoc: true
 homepage: https://github.com/danslimmon/apachecrunch/
 licenses: 
@@ -55,27 +66,21 @@ rdoc_options: []
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
+  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
+  version: 
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.6.2
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: Apache log analysis tool designed for ease of use

data/lib/log_element.rb

@@ -1,351 +0,0 @@
-# Converts a string to an integer
-class IntegerCast
-    def self.cast(string_value)
-        string_value.to_i
-    end
-end
-
-
-# Converts a CLF-formatted string to an integer
-#
-# "CLF-formatted" means that if the value is 0, the string will be a single hyphen instead of
-# a number.  Like %b, for instance.
-class CLFIntegerCast
-    def self.cast(string_value)
-        if string_value == "-"
-            return 0
-        end
-        string_value.to_i
-    end
-end
-
-
-# An element in a log format.  Abstract from which all elements inherit.
-#
-# Exposes:
-#    abbrev: The Apache abbreviation for the element (such as "%h" or "%u" or "%{Referer}i")
-#    name: A short name for the element (such as "remote_host", "remote_user", or "reqhead_referer")
-#    regex: A regex that should match such an element ("[A-Za-z0-9.-]+", "[^:]+", ".+")
-#
-# If '_caster' is not nil, it should be a class with a method called "cast" that
-# transforms a string to the appropriate data type or format for consumption.
-# For example, the IntegerCast class transforms "562" to 562.  The correct cast
-# of a string can then be performed by passing that string to this LogFormaElement
-# instance's "cast" method.
-#
-# 'derive_elements' manages elements that can be derived from the instance's value.  See
-# ReqFirstlineElement for an example.
-class LogFormatElement
-    @_caster = nil
-
-    attr_accessor :abbrev, :name, :regex
-    # Class variables that determine the _default_ for abbrev, name, and regex in an instance.
-    # That is, an instance will initialize with these values for the instance variables @abbrev,
-    # @name, and @regex.
-    class << self; attr_accessor :abbrev, :name, :regex end
-    # Additionally we need to access this from within the instance:
-    class << self; attr_accessor :_caster end
-
-    def initialize
-        @abbrev = self.class.abbrev
-        @name = self.class.name
-        @regex = self.class.regex
-    end
-
-    # Casts a string found in the log to the correct type, using the class's @_caster attribute.
-    def cast(string_value)
-        if _caster.nil?
-            return string_value
-        else
-            return _caster.cast(string_value)
-        end
-    end
-
-    # Derives the named element (e.g. "url_path") from a given value for this one.
-    #
-    # See ReqFirstlineElement for an example.
-    def self.derive(name, our_own_value)
-        raise NotImplementedError
-    end
-
-    # Returns a list of the element classes that can be derived from this one.
-    #
-    # See ReqFirstlineElement for an example.
-    def derived_elements
-        []
-    end
-end
-
-
-class RemoteHostElement < LogFormatElement
-    @abbrev = "%h"
-    @name = :remote_host
-    @regex = %q![A-Za-z0-9.-]+!
-end
-
-
-class LogNameElement < LogFormatElement
-    @abbrev = "%l"
-    @name = :log_name
-    @regex = %q!\S+!
-end
-
-
-class RemoteUserElement < LogFormatElement
-    @abbrev = "%u"
-    @name = :remote_user
-    @regex = %q![^:]+!
-end
-
-
-class TimeElement < LogFormatElement
-    @abbrev = "%t"
-    @name = :time
-    @regex = %q!\[\d\d/[A-Za-z]{3}/\d\d\d\d:\d\d:\d\d:\d\d [-+]\d\d\d\d\]!
-
-    @_derivation_regex = nil
-    @_month_map = {"Jan" => 1, "Feb" => 2, "Mar" => 3, "Apr" => 4, "May" => 5, "Jun" => 6,
-                   "Jul" => 7, "Aug" => 8, "Sep" => 9, "Oct" => 10, "Nov" => 11, "Dec" => 12}
-
-    def self.derive(name, our_own_value)
-        if @_derivation_regex.nil?
-            @_derivation_regex = Regexp.compile(%q!^\[(\d\d)/([A-Za-z]{3})/(\d\d\d\d):(\d\d):(\d\d):(\d\d)!)
-        end
-
-        hsh = {}
-        if our_own_value =~ @_derivation_regex
-            hsh[:year] = $3.to_i
-            hsh[:month] = @_month_map[$2]
-            hsh[:day] = $1.to_i
-
-            hsh[:hour] = $4.to_i
-            hsh[:minute] = $5.to_i
-            hsh[:second] = $6.to_i
-        end
-
-        hsh[name]
-    end
-
-    def derived_elements
-        [YearElement, MonthElement, DayElement, HourElement, MinuteElement, SecondElement]
-    end
-end
-
-
-# Elements derived from TimeElement
-class YearElement < LogFormatElement
-    @name = :year
-    @regex = %q!\d{4}!
-end
-class MonthElement < LogFormatElement
-    @name = :month
-    @regex = %q![A-Za-z]{3}!
-end
-class DayElement < LogFormatElement
-    @name = :day
-    @regex = %q!\d{2}!
-end
-class HourElement < LogFormatElement
-    @name = :hour
-    @regex = %q!\d{2}!
-end
-class MinuteElement < LogFormatElement
-    @name = :minute
-    @regex = %q!\d{2}!
-end
-class SecondElement < LogFormatElement
-    @name = :second
-    @regex = %q!\d{2}!
-end
-
-
-class ReqFirstlineElement < LogFormatElement
-    @abbrev = "%r"
-    @name = :req_firstline
-    @regex = %q![^"]+!
-
-    @_derivation_regex = nil
-
-    def self.derive(name, our_own_value)
-        if @_derivation_regex.nil?
-            @_derivation_regex = Regexp.compile("^(#{ReqMethodElement.regex})\s+(#{UrlPathElement.regex})(#{QueryStringElement.regex})\s+(#{ProtocolElement.regex})$")
-        end
-
-        hsh = {}
-        if our_own_value =~ @_derivation_regex
-            hsh[ReqMethodElement.name] = $1
-            hsh[UrlPathElement.name] = $2
-            hsh[QueryStringElement.name] = $3
-            hsh[ProtocolElement.name] = $4
-        end
-
-        hsh[name]
-    end
-
-    def derived_elements
-        return [ReqMethodElement, UrlPathElement, QueryStringElement, ProtocolElement]
-    end
-end
-
-
-class StatusElement < LogFormatElement
-    @abbrev = "%s"
-    @name = :status
-    @regex = %q!\d+|-!
-end
-
-
-class BytesSentElement < LogFormatElement
-    @abbrev = "%b"
-    @name = :bytes_sent
-    @regex = %q!\d+!
-
-    @@_caster = IntegerCast
-end
-
-
-class BytesSentElement < LogFormatElement
-    @abbrev = "%b"
-    @name = :bytes_sent
-    @regex = %q![\d-]+!
-
-    @_caster = CLFIntegerCast
-end
-
-
-class BytesSentWithHeadersElement < LogFormatElement
-    @abbrev = "%O"
-    @name = :bytes_sent_with_headers
-    @regex = %q!\d+!
-
-    @_caster = IntegerCast
-end
-
-
-class ServeTimeMicroElement < LogFormatElement
-    @abbrev = "%D"
-    @name = :serve_time_micro
-    @regex = %q!\d+!
-
-    @_caster = IntegerCast
-end
-
-
-class UrlPathElement < LogFormatElement
-    @abbrev = "%U"
-    @name = :url_path
-    @regex = %q!/[^?]*!
-end
-
-
-class QueryStringElement < LogFormatElement
-    @abbrev = "%q"
-    @name = :query_string
-    @regex = %q!\??\S*!
-end
-
-
-class ReqMethodElement < LogFormatElement
-    @abbrev = "%m"
-    @name = :req_method
-    @regex = %q![A-Z]+!
-end
-
-
-class ProtocolElement < LogFormatElement
-    @abbrev = "%H"
-    @name = :protocol
-    @regex = %q!\S+!
-end
-
-
-class ReqheaderElement < LogFormatElement
-end
-
-
-class RegexElement < LogFormatElement
-end
-
-
-# Finds log format elements given information about them.
-class ElementDictionary
-    @@_ELEMENTS = [
-                    RemoteHostElement,
-                    LogNameElement,
-                    RemoteUserElement,
-                    TimeElement,
-                    ReqFirstlineElement,
-                    StatusElement,
-                    BytesSentElement,
-                    BytesSentElement,
-                    BytesSentWithHeadersElement,
-                    ServeTimeMicroElement,
-                    UrlPathElement,
-                    QueryStringElement,
-                    ReqMethodElement,
-                    ProtocolElement
-    ]
-
-    # Returns the LogFormatElement subclass with the given format-string abbreviation.
-    #
-    # If none exists, returns nil.
-    def self.find_by_abbrev(abbrev)
-        @@_ELEMENTS.each do |element|
-            if element.abbrev == abbrev
-                return element
-            end
-        end
-
-        nil
-    end
-end
-
-
-# Generates LogFormatElement instances.
-#
-# This class does the work of figuring out which LogFormatElement subclass to make and makes it.
-class LogFormatElementFactory
-    # Takes an Apache log format abbreviation and returns a corresponding LogFormatElement
-    def from_abbrev(abbrev)
-        element_cls = ElementDictionary.find_by_abbrev(abbrev)
-        if element_cls
-            # We found it in the dictionary, so just return an instance
-            return element_cls.new
-        elsif abbrev =~ /^%\{([A-Za-z0-9-]+)\}i/
-            # HTTP request header
-            return _reqheader_element(abbrev, $1)
-        elsif abbrev =~ /^%\{(.*?):([^}]+)\}r/
-            # Arbitrary regex
-            return _regex_element(abbrev, $1, $2)
-        end
-
-        raise "Unknown element format '#{abbrev}'"
-    end
-
-    # Returns a format element based on an HTTP header
-    def _reqheader_element(abbrev, header_name)
-        element = ReqheaderElement.new
-
-        element.abbrev = abbrev
-        element.regex = %q![^"]*!
-        element.name = _header_name_to_element_name(header_name)
-
-        element
-    end
-
-    # Returns a format element based on an arbitrary regex
-    def _regex_element(abbrev, regex_name, regex)
-        element = RegexElement.new
-
-        element.abbrev = abbrev
-        element.regex = regex
-        element.name = "regex_#{regex_name}".to_sym
-
-        element
-    end
-
-    # Lowercases header name and turns hyphens into underscores
-    def _header_name_to_element_name(header_name)
-        ("reqheader_" + header_name.downcase().gsub("-", "_")).to_sym
-    end
-end