jls-grok 0.1.2786

data/sample.rb

@@ -0,0 +1,43 @@
+require "Grok"
+require "pp"
+
+patterns = {}
+
+matches = [
+  #"%{SYSLOGBASE} Accepted %{NOTSPACE:method} for %{DATA:user} from %{IPORHOST:client} port %{INT:port}",
+  #"%{SYSLOGBASE} Did not receive identification string from %{IPORHOST:client}",
+  #"%{SYSLOGBASE} error: PAM: authentication error for %{DATA:user} from %{IPORHOST:client}",
+  "%{SYSLOGBASE} .*"
+  #"%{COMBINEDAPACHELOG}",
+  #"%{UNINDEXED}hello (?=%{GREEDYDATA})%{WORD}"
+  
+  #"( *%{DATA:key}:%{NOTSPACE:value})+"
+]
+
+groks = matches.collect do |m|
+  g = Grok.new
+  g.add_patterns_from_file("../patterns/base")
+  g.compile(m)
+  g
+end
+
+bytes = 0
+time_start = Time.now.to_f
+$stdin.each do |line|
+  groks.each do |grok|
+    m = grok.match(line)
+    if m
+      #data = Hash.new { |h,k| h[k] = Array.new }
+      #m.each_capture do |key, value|
+        #data[key] << value
+      #end
+      #pp data
+      pp m.captures
+      #bytes += line.length
+      break
+    end
+  end
+end
+
+#time_end = Time.now.to_f
+#puts "parse rate: #{ (bytes / 1024) / (time_end - time_start) }"

data/test/GDB_COMMAND

@@ -0,0 +1,29 @@
+set confirm off
+delete 
+
+break rGrokMatch_new_from_grok_match if strcmp(gm->subject, "40079") == 0
+commands
+printf "subject: %s\n", gm->subject
+delete 
+break rGrokMatch_captures
+cont
+end
+
+set confirm off
+delete 
+break rGrokMatch_new_from_grok_match if strcmp(gm->subject, "-29086") == 0
+commands
+print *gm
+delete
+break rGrokMatch_captures
+cont
+end
+
+run
+
+next
+print *gm
+print *(gm->grok)
+delete
+cont
+

data/test/Makefile

@@ -0,0 +1,7 @@
+
+.PHONY: test
+test:
+	$(MAKE) -C ../../ libgrok.so
+	[ -f "../Makefile" ] || (cd ../ext; ruby extconf.rb)
+	$(MAKE) -C ../ext Grok.so
+	LD_LIBRARY_PATH="$${LD_LIBRARY_PATH}:$$PWD/../../" RUBYLIB="$$PWD/../ext" ruby alltests.rb

data/test/alltests.rb

@@ -0,0 +1,6 @@
+require 'test/unit'
+
+Dir["#{File.dirname(__FILE__)}/*/**/*.rb"].each do |file|
+  require file
+end
+

data/test/general/basic_test.rb

@@ -0,0 +1,58 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class GrokBasicTests < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+  end
+
+  def test_grok_methods
+    assert_respond_to(@grok, :compile)
+    assert_respond_to(@grok, :match)
+    assert_respond_to(@grok, :expanded_pattern)
+    assert_respond_to(@grok, :pattern)
+  end
+
+  def test_grok_compile_fails_on_invalid_expressions
+    bad_regexps = ["[", "[foo", "?", "foo????", "(?-"]
+    bad_regexps.each do |regexp|
+      assert_raise ArgumentError do
+        @grok.compile(regexp)
+      end
+    end
+  end
+
+  def test_grok_compile_succeeds_on_valid_expressions
+    good_regexps = ["[hello]", "(test)", "(?:hello)", "(?=testing)"]
+    good_regexps.each do |regexp|
+      assert_nothing_raised do
+        @grok.compile(regexp)
+      end
+    end
+  end
+
+  def test_grok_pattern_is_same_as_compile_pattern
+    pattern = "Hello world"
+    @grok.compile(pattern)
+    assert_equal(pattern, @grok.pattern)
+  end
+
+  # TODO(sissel): Move this test to a separate test suite aimed
+  # at testing grok internals
+  def test_grok_expanded_pattern_works_correctly
+    @grok.add_pattern("test", "hello world")
+    @grok.compile("%{test}")
+    assert_equal("(?<0000>hello world)", @grok.expanded_pattern)
+  end
+
+  def test_grok_load_patterns_from_file
+    require 'tempfile'
+    fd = Tempfile.new("grok_test_patterns.XXXXX")
+    fd.puts "TEST \\d+"
+    fd.close
+    @grok.add_patterns_from_file(fd.path)
+    @grok.compile("%{TEST}")
+    assert_equal("(?<0000>\\d+)", @grok.expanded_pattern)
+  end
+end

data/test/general/captures_test.rb

@@ -0,0 +1,88 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class GrokPatternCapturingTests < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+  end
+
+  def test_capture_methods
+    @grok.add_pattern("foo", ".*")
+    @grok.compile("%{foo}")
+    match = @grok.match("hello world")
+    assert_respond_to(match, :captures)
+    assert_respond_to(match, :start)
+    assert_respond_to(match, :end)
+    assert_respond_to(match, :subject)
+    assert_respond_to(match, :each_capture)
+  end
+
+  def test_basic_capture
+    @grok.add_pattern("foo", ".*")
+    @grok.compile("%{foo}")
+    input = "hello world"
+    match = @grok.match(input)
+    assert_equal("(?<0000>.*)", @grok.expanded_pattern)
+    assert_kind_of(GrokMatch, match)
+    assert_kind_of(Hash, match.captures)
+    assert_equal(match.captures.length, 1)
+    assert_kind_of(Array, match.captures["foo"])
+    assert_equal(1, match.captures["foo"].length)
+    assert_kind_of(String, match.captures["foo"][0])
+    assert_equal(input, match.captures["foo"][0])
+
+    assert_kind_of(Fixnum, match.start)
+    assert_kind_of(Fixnum, match.end)
+    assert_kind_of(String, match.subject)
+    assert_equal(0, match.start,
+                 "Match of /.*/, start should equal 0")
+    assert_equal(input.length, match.end,
+                 "Match of /.*/, end should equal input string length")
+    assert_equal(input, match.subject)
+  end
+
+  def test_multiple_captures_with_same_name
+    @grok.add_pattern("foo", "\\w+")
+    @grok.compile("%{foo} %{foo}")
+    match = @grok.match("hello world")
+    assert_not_equal(false, match)
+    assert_equal(1, match.captures.length)
+    assert_equal(2, match.captures["foo"].length)
+    assert_equal("hello", match.captures["foo"][0])
+    assert_equal("world", match.captures["foo"][1])
+  end
+
+  def test_multiple_captures
+    @grok.add_pattern("foo", "\\w+")
+    @grok.add_pattern("bar", "\\w+")
+    @grok.compile("%{foo} %{bar}")
+    match = @grok.match("hello world")
+    assert_not_equal(false, match)
+    assert_equal(2, match.captures.length)
+    assert_equal(1, match.captures["foo"].length)
+    assert_equal(1, match.captures["bar"].length)
+    assert_equal("hello", match.captures["foo"][0])
+    assert_equal("world", match.captures["bar"][0])
+  end
+
+  def test_nested_captures
+    @grok.add_pattern("foo", "\\w+ %{bar}")
+    @grok.add_pattern("bar", "\\w+")
+    @grok.compile("%{foo}")
+    match = @grok.match("hello world")
+    assert_not_equal(false, match)
+    assert_equal(2, match.captures.length)
+    assert_equal(1, match.captures["foo"].length)
+    assert_equal(1, match.captures["bar"].length)
+    assert_equal("hello world", match.captures["foo"][0])
+    assert_equal("world", match.captures["bar"][0])
+  end
+
+  def test_nesting_recursion
+    @grok.add_pattern("foo", "%{foo}")
+    assert_raises(ArgumentError) do
+      @grok.compile("%{foo}")
+    end
+  end
+end