jls-grok 0.4.7 → 0.5.2

data/test/patterns/ip.rb

@@ -1,31 +0,0 @@
-require 'grok'
-require 'test/unit'
-
-class IPPatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-  end
-
-  def test_ips
-    @grok.compile("%{IP}")
-    File.open("#{File.dirname(__FILE__)}/ip.input").each do |line|
-      line.chomp!
-      match = @grok.match(line)
-      assert_not_equal(false, match)
-      assert_equal(line, match.captures["IP"][0])
-    end
-  end
-
-  def test_non_ips
-    @grok.compile("%{IP}")
-    nonips = %w{255.255.255.256 0.1.a.33 300.1.2.3 300 400.4.3.a 1.2.3.b
-                1..3.4.5 hello world}
-    nonips << "hello world"
-    nonips.each do |input|
-      match = @grok.match(input)
-      assert_equal(false, match)
-    end
-  end
-end

data/test/patterns/iso8601.rb

@@ -1,68 +0,0 @@
-require 'grok'
-require 'test/unit'
-
-class ISO8601PatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-    @grok.compile("^%{TIMESTAMP_ISO8601}$")
-  end
-
-  def test_iso8601
-    times = [
-      "2001-01-01T00:00:00",
-      "1974-03-02T04:09:09",
-      "2010-05-03T08:18:18+00:00",
-      "2004-07-04T12:27:27-00:00",
-      "2001-09-05T16:36:36+0000",
-      "2001-11-06T20:45:45-0000",
-      "2001-12-07T23:54:54Z",
-      "2001-01-01T00:00:00.123456",
-      "1974-03-02T04:09:09.123456",
-      "2010-05-03T08:18:18.123456+00:00",
-      "2004-07-04T12:27:27.123456-00:00",
-      "2001-09-05T16:36:36.123456+0000",
-      "2001-11-06T20:45:45.123456-0000",
-      "2001-12-07T23:54:54.123456Z",
-      "2001-12-07T23:54:60.123456Z", # '60' second is a leap second.
-    ]
-    times.each do |time|
-      match = @grok.match(time)
-      assert_not_equal(false, match, "Expected #{time} to match TIMESTAMP_ISO8601")
-      assert_equal(time, match.captures["TIMESTAMP_ISO8601"][0])
-    end
-  end
-
-  def test_iso8601_nomatch
-    times = [
-      "2001-13-01T00:00:00", # invalid month
-      "2001-00-01T00:00:00", # invalid month
-      "2001-01-00T00:00:00", # invalid day
-      "2001-01-32T00:00:00", # invalid day
-      "2001-01-aT00:00:00", # invalid day
-      "2001-01-1aT00:00:00", # invalid day
-      "2001-01-01Ta0:00:00", # invalid hour
-      "2001-01-01T0:00:00", # invalid hour
-      "2001-01-01T25:00:00", # invalid hour
-      "2001-01-01T01:60:00", # invalid minute
-      "2001-01-01T00:aa:00", # invalid minute
-      "2001-01-01T00:00:aa", # invalid second
-      "2001-01-01T00:00:-1", # invalid second
-      "2001-01-01T00:00:61", # invalid second
-      "2001-01-01T00:00:00A", # invalid timezone
-      "2001-01-01T00:00:00+", # invalid timezone
-      "2001-01-01T00:00:00+25", # invalid timezone
-      "2001-01-01T00:00:00+2500", # invalid timezone
-      "2001-01-01T00:00:00+25:00", # invalid timezone
-      "2001-01-01T00:00:00-25", # invalid timezone
-      "2001-01-01T00:00:00-2500", # invalid timezone
-      "2001-01-01T00:00:00-00:61", # invalid timezone
-    ]
-    times.each do |time|
-      match = @grok.match(time)
-      assert_equal(false, match, "Expected #{time} to not match TIMESTAMP_ISO8601")
-    end
-  end
-
-end

data/test/patterns/month.rb

@@ -1,24 +0,0 @@
-require 'grok'
-require 'test/unit'
-
-class MonthPatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-    @grok.compile("%{MONTH}")
-  end
-
-  def test_months
-    months = ["Jan", "January", "Feb", "February", "Mar", "March", "Apr",
-              "April", "May", "Jun", "June", "Jul", "July", "Aug", "August",
-              "Sep", "September", "Oct", "October", "Nov", "November", "Dec",
-              "December"]
-    months.each do |month|
-      match = @grok.match(month)
-      assert_not_equal(false, match, "Expected #{month} to match")
-      assert_equal(month, match.captures["MONTH"][0])
-    end
-  end
-
-end

data/test/patterns/number.rb

@@ -1,69 +0,0 @@
-require 'grok'
-require 'test/unit'
-
-class NumberPatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-  end
-
-  def test_match_number
-    @grok.compile("%{NUMBER}")
-    # step of a prime number near 100 so we get about 2000 iterations
-    #puts @grok.expanded_pattern.inspect
-    -100000.step(100000, 97) do |value|
-      match = @grok.match(value.to_s)
-      assert_not_equal(false, match, "#{value} should not match false")
-      assert_equal(value.to_s, match.captures["NUMBER"][0])
-    end
-  end
-
-  def test_match_number_float
-    # generate some random floating point values
-    # always seed with the same random number, so the test is always the same
-    srand(0)
-    @grok.compile("%{NUMBER}")
-    0.upto(1000) do |value|
-      value = (rand * 100000 - 50000).to_s
-      match = @grok.match(value)
-      assert_not_equal(false, match)
-      assert_equal(value, match.captures["NUMBER"][0])
-    end
-  end
-
-  def test_match_number_amid_things
-    @grok.compile("%{NUMBER}")
-    value = "hello 12345 world"
-    match = @grok.match(value)
-    assert_not_equal(false, match)
-    assert_equal("12345", match.captures["NUMBER"][0])
-
-    value = "Something costs $55.4!"
-    match = @grok.match(value)
-    assert_not_equal(false, match)
-    assert_equal("55.4", match.captures["NUMBER"][0])
-  end
-
-  def test_no_match_number
-    @grok.compile("%{NUMBER}")
-    ["foo", "", " ", ".", "hello world", "-abcd"].each do |value|
-      match = @grok.match(value.to_s)
-      assert_equal(false, match)
-    end
-  end
-
-  def test_match_base16num
-    @grok.compile("%{BASE16NUM}")
-    # Ruby represents negative values in a strange way, so only
-    # test positive numbers for now.
-    # I don't think anyone uses negative values in hex anyway...
-    0.upto(1000) do |value|
-      [("%x" % value), ("0x%08x" % value), ("%016x" % value)].each do |hexstr|
-        match = @grok.match(hexstr)
-        assert_not_equal(false, match)
-        assert_equal(hexstr, match.captures["BASE16NUM"][0])
-      end
-    end
-  end
-end

data/test/patterns/path.rb

@@ -1,31 +0,0 @@
-require 'grok'
-require 'test/unit'
-
-class PathPatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-    @grok.compile("%{PATH}")
-  end
-
-  def test_unix_paths
-    paths = %w{/ /usr /usr/bin /usr/bin/foo /etc/motd /home/.test
-               /foo/bar//baz //testing /.test /%foo% /asdf/asdf,v}
-    paths.each do |path|
-      match = @grok.match(path)
-      assert_not_equal(false, match)
-      assert_equal(path, match.captures["PATH"][0])
-    end
-  end
-
-  def test_windows_paths
-    paths = %w{C:\WINDOWS \\\\Foo\bar \\\\1.2.3.4\C$ \\\\some\path\here.exe}
-    paths << "C:\\Documents and Settings\\"
-    paths.each do |path|
-      match = @grok.match(path)
-      assert_not_equal(false, match, "Expected #{path} to match, but it didn't.")
-      assert_equal(path, match.captures["PATH"][0])
-    end
-  end
-end

data/test/patterns/prog.rb

@@ -1,21 +0,0 @@
-require 'grok'
-require 'test/unit'
-
-class ProgPatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-    @grok.compile("^%{PROG}$")
-  end
-
-  def test_progs
-    progs = %w{kernel foo-bar foo_bar foo/bar/baz}
-    progs.each do |prog|
-      match = @grok.match(prog)
-      assert_not_equal(false, prog, "Expected #{prog} to match.")
-      assert_equal(prog, match.captures["PROG"][0], "Expected #{prog} to match capture.")
-    end
-  end
-
-end

data/test/patterns/quotedstring.rb

@@ -1,54 +0,0 @@
-#require 'rubygems'
-require 'grok'
-require 'test/unit'
-
-class QuotedStringPatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-  end
-
-  def test_quoted_string_common
-    @grok.compile("%{QUOTEDSTRING}")
-    inputs = ["hello", ""]
-    quotes = %w{" ' `}
-    inputs.each do |value|
-      quotes.each do |quote|
-        str = "#{quote}#{value}#{quote}"
-        match = @grok.match(str)
-        assert_not_equal(false, match)
-        assert_equal(str, match.captures["QUOTEDSTRING"][0])
-      end
-    end
-  end
-
-  def test_quoted_string_inside_escape
-    @grok.compile("%{QUOTEDSTRING}")
-    quotes = %w{" ' `}
-    quotes.each do |quote|
-      str = "#{quote}hello \\#{quote}world\\#{quote}#{quote}"
-      match = @grok.match(str)
-      assert_not_equal(false, match)
-      assert_equal(str, match.captures["QUOTEDSTRING"][0])
-    end
-  end
-
-  def test_escaped_quotes_no_match_quoted_string
-    @grok.compile("%{QUOTEDSTRING}")
-    inputs = ["\\\"testing\\\"", "\\\'testing\\\'", "\\\`testing\\\`",]
-    inputs.each do |value|
-      match = @grok.match(value)
-      assert_equal(false, match)
-    end
-  end
-
-  def test_non_quoted_strings_no_match
-    @grok.compile("%{QUOTEDSTRING}")
-    inputs = ["\\\"testing", "testing", "hello world ' something ` foo"]
-    inputs.each do |value|
-      match = @grok.match(value)
-      assert_equal(false, match)
-    end
-  end
-end

data/test/patterns/uri.rb

@@ -1,46 +0,0 @@
-require 'grok'
-require 'test/unit'
-
-class URIPatternsTest < Test::Unit::TestCase
-  def setup
-    @grok = Grok.new
-    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
-    @grok.add_patterns_from_file(path)
-    @grok.compile("%{URI}")
-  end
-
-  def test_urls
-    urls = ["http://www.google.com", "telnet://helloworld",
-            "http://www.example.com/", "http://www.example.com/test.html", 
-            "http://www.example.com/test.html?foo=bar", 
-            "http://www.example.com/test.html?foo=bar&fizzle=baz", 
-            "http://www.example.com:80/test.html?foo=bar&fizzle=baz", 
-            "https://www.example.com:443/test.html?foo=bar&fizzle=baz", 
-            "https://user@www.example.com:443/test.html?foo=bar&fizzle=baz", 
-            "https://user:pass@somehost/fetch.pl",
-            "puppet:///",
-            "http://www.foo.com",
-            "http://www.foo.com/",
-            "http://www.foo.com/?testing",
-            "http://www.foo.com/?one=two",
-            "http://www.foo.com/?one=two&foo=bar",
-            "foo://somehost.com:12345",
-            "foo://user@somehost.com:12345",
-            "foo://user@somehost.com:12345/",
-            "foo://user@somehost.com:12345/foo.bar/baz/fizz",
-            "foo://user@somehost.com:12345/foo.bar/baz/fizz?test",
-            "foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4",
-            "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search",
-            "http://www.freebsd.org/cgi/url.cgi?ports/sysutils/grok/pkg-descr",
-            "http://www.google.com/search?q=CAPTCHA+ssh&start=0&ie=utf-8&oe=utf-8&client=firefox-a&rls=org.mozilla:en-US:official",
-            "svn+ssh://somehost:12345/testing",
-           ]
-            
-    urls.each do |url|
-      match = @grok.match(url)
-      assert_not_equal(false, match, "Expected this to match: #{url}")
-      assert_equal(url, match.captures["URI"][0])
-    end
-  end
-
-end

data/test/run.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$PWD/../../" RUBYLIB="$PWD/../ext:$PWD/../lib" ruby "$@"

data/test/speedtest.rb

@@ -1,58 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'grok'
-#require 'ruby-prof'
-require 'pp'
-
-#RubyProf.start  
-
-iterations = 100000
-pattern = "[A-z0-9_-]*\\[[0-9]+\\]"
-
-grok = Grok.new
-grok.add_patterns_from_file("../../patterns/base")
-grok.compile("%{COMBINEDAPACHELOG}")
-
-#rubyre = Regexp.new("(?<foo>#{pattern})")
-#rubyre = Regexp.new(pattern)
-
-matches = { :grok => 0, :rubyre => 0 }
-failures = { :grok => 0, :rubyre => 0 }
-def time(iterations, &block)
-  start = Time.now
-  file = File.open("/b/logs/access")
-  data = (1 .. iterations).collect { file.readline() }
-  data.each do |line|
-    block.call(line)
-  end
-  return Time.now - start
-end
-
-groktime = time(iterations) do |line|
-  m = grok.match(line)
-  if m
-    matches[:grok] += 1
-    m.captures["FOO"]
-  else
-    #puts line
-    failures[:grok] +=1
-  end
-end
-
-#rubyretime = time(iterations) do |line|
-  #m = rubyre.match(line)
-  #if m 
-    #matches[:rubyre] += 1
-    #m["foo"]
-  #end
-#end
-
-puts "Grok: #{matches[:grok] / groktime}"
-puts matches.inspect
-puts failures.inspect
-#puts "rubyre: #{rubyretime}"
-#puts matches.inspect
-#result = RubyProf.stop
-#printer = RubyProf::FlatPrinter.new(result)
-#printer.print(STDOUT, 0)