jls-grok 0.1.2786

data/test/patterns/ip.rb

@@ -0,0 +1,32 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class IPPatternsTest < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
+    @grok.add_patterns_from_file(path)
+  end
+
+  def test_ips
+    @grok.compile("%{IP}")
+    File.open("#{File.dirname(__FILE__)}/ip.input").each do |line|
+      line.chomp!
+      match = @grok.match(line)
+      assert_not_equal(false, match)
+      assert_equal(line, match.captures["IP"][0])
+    end
+  end
+
+  def test_non_ips
+    @grok.compile("%{IP}")
+    nonips = %w{255.255.255.256 0.1.a.33 300.1.2.3 300 400.4.3.a 1.2.3.b
+                1..3.4.5 hello world}
+    nonips << "hello world"
+    nonips.each do |input|
+      match = @grok.match(input)
+      assert_equal(false, match)
+    end
+  end
+end

data/test/patterns/month.rb

@@ -0,0 +1,25 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class MonthPatternsTest < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
+    @grok.add_patterns_from_file(path)
+    @grok.compile("%{MONTH}")
+  end
+
+  def test_urls
+    months = ["Jan", "January", "Feb", "February", "Mar", "March", "Apr",
+              "April", "May", "Jun", "June", "Jul", "July", "Aug", "August",
+              "Sep", "September", "Oct", "October", "Nov", "November", "Dec",
+              "December"]
+    months.each do |month|
+      match = @grok.match(month)
+      assert_not_equal(false, match)
+      assert_equal(month, match.captures["MONTH"][0])
+    end
+  end
+
+end

data/test/patterns/number.rb

@@ -0,0 +1,70 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class NumberPatternsTest < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
+    @grok.add_patterns_from_file(path)
+  end
+
+  def test_match_number
+    @grok.compile("%{NUMBER}")
+    # step of a prime number near 100 so we get about 2000 iterations
+    #puts @grok.expanded_pattern.inspect
+    -100000.step(100000, 97) do |value|
+      match = @grok.match(value.to_s)
+      assert_not_equal(false, match, "#{value} should not match false")
+      assert_equal(value.to_s, match.captures["NUMBER"][0])
+    end
+  end
+
+  def test_match_number_float
+    # generate some random floating point values
+    # always seed with the same random number, so the test is always the same
+    srand(0)
+    @grok.compile("%{NUMBER}")
+    0.upto(1000) do |value|
+      value = (rand * 100000 - 50000).to_s
+      match = @grok.match(value)
+      assert_not_equal(false, match)
+      assert_equal(value, match.captures["NUMBER"][0])
+    end
+  end
+
+  def test_match_number_amid_things
+    @grok.compile("%{NUMBER}")
+    value = "hello 12345 world"
+    match = @grok.match(value)
+    assert_not_equal(false, match)
+    assert_equal("12345", match.captures["NUMBER"][0])
+
+    value = "Something costs $55.4!"
+    match = @grok.match(value)
+    assert_not_equal(false, match)
+    assert_equal("55.4", match.captures["NUMBER"][0])
+  end
+
+  def test_no_match_number
+    @grok.compile("%{NUMBER}")
+    ["foo", "", " ", ".", "hello world", "-abcd"].each do |value|
+      match = @grok.match(value.to_s)
+      assert_equal(false, match)
+    end
+  end
+
+  def test_match_base16num
+    @grok.compile("%{BASE16NUM}")
+    # Ruby represents negative values in a strange way, so only
+    # test positive numbers for now.
+    # I don't think anyone uses negative values in hex anyway...
+    0.upto(1000) do |value|
+      [("%x" % value), ("0x%08x" % value), ("%016x" % value)].each do |hexstr|
+        match = @grok.match(hexstr)
+        assert_not_equal(false, match)
+        assert_equal(hexstr, match.captures["BASE16NUM"][0])
+      end
+    end
+  end
+end

data/test/patterns/path.rb

@@ -0,0 +1,32 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class PathPatternsTest < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
+    @grok.add_patterns_from_file(path)
+    @grok.compile("%{PATH}")
+  end
+
+  def test_unix_paths
+    paths = %w{/ /usr /usr/bin /usr/bin/foo /etc/motd /home/.test
+               /foo/bar//baz //testing /.test /%foo% /asdf/asdf,v}
+    paths.each do |path|
+      match = @grok.match(path)
+      assert_not_equal(false, match)
+      assert_equal(path, match.captures["PATH"][0])
+    end
+  end
+
+  def test_windows_paths
+    paths = %w{C:\WINDOWS \\Foo\bar}
+    paths << "C:\\Documents and Settings\\"
+    paths.each do |path|
+      match = @grok.match(path)
+      assert_not_equal(false, match)
+      assert_equal(path, match.captures["PATH"][0])
+    end
+  end
+end

data/test/patterns/quotedstring.rb

@@ -0,0 +1,54 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class QuotedStringPatternsTest < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
+    @grok.add_patterns_from_file(path)
+  end
+
+  def test_quoted_string_common
+    @grok.compile("%{QUOTEDSTRING}")
+    inputs = ["hello", ""]
+    quotes = %w{" ' `}
+    inputs.each do |value|
+      quotes.each do |quote|
+        str = "#{quote}#{value}#{quote}"
+        match = @grok.match(str)
+        assert_not_equal(false, match)
+        assert_equal(str, match.captures["QUOTEDSTRING"][0])
+      end
+    end
+  end
+
+  def test_quoted_string_inside_escape
+    @grok.compile("%{QUOTEDSTRING}")
+    quotes = %w{" ' `}
+    quotes.each do |quote|
+      str = "#{quote}hello \\#{quote}world\\#{quote}#{quote}"
+      match = @grok.match(str)
+      assert_not_equal(false, match)
+      assert_equal(str, match.captures["QUOTEDSTRING"][0])
+    end
+  end
+
+  def test_escaped_quotes_no_match_quoted_string
+    @grok.compile("%{QUOTEDSTRING}")
+    inputs = ["\\\"testing\\\"", "\\\'testing\\\'", "\\\`testing\\\`",]
+    inputs.each do |value|
+      match = @grok.match(value)
+      assert_equal(false, match)
+    end
+  end
+
+  def test_non_quoted_strings_no_match
+    @grok.compile("%{QUOTEDSTRING}")
+    inputs = ["\\\"testing", "testing", "hello world ' something ` foo"]
+    inputs.each do |value|
+      match = @grok.match(value)
+      assert_equal(false, match)
+    end
+  end
+end

data/test/patterns/uri.rb

@@ -0,0 +1,44 @@
+#require 'rubygems'
+require 'Grok'
+require 'test/unit'
+
+class URIPatternsTest < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
+    @grok.add_patterns_from_file(path)
+    @grok.compile("%{URI}")
+  end
+
+  def test_urls
+    urls = ["http://www.google.com", "telnet://helloworld",
+            "http://www.example.com/", "http://www.example.com/test.html", 
+            "http://www.example.com/test.html?foo=bar", 
+            "http://www.example.com/test.html?foo=bar&fizzle=baz", 
+            "http://www.example.com:80/test.html?foo=bar&fizzle=baz", 
+            "https://www.example.com:443/test.html?foo=bar&fizzle=baz", 
+            "https://user@www.example.com:443/test.html?foo=bar&fizzle=baz", 
+            "https://user:pass@somehost/fetch.pl",
+            "puppet:///",
+            "http://www.foo.com",
+            "http://www.foo.com/",
+            "http://www.foo.com/?testing",
+            "http://www.foo.com/?one=two",
+            "http://www.foo.com/?one=two&foo=bar",
+            "foo://somehost.com:12345",
+            "foo://user@somehost.com:12345",
+            "foo://user@somehost.com:12345/",
+            "foo://user@somehost.com:12345/foo.bar/baz/fizz",
+            "foo://user@somehost.com:12345/foo.bar/baz/fizz?test",
+            "foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4",
+            "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search"
+           ]
+            
+    urls.each do |url|
+      match = @grok.match(url)
+      assert_not_equal(false, match)
+      assert_equal(url, match.captures["URI"][0])
+    end
+  end
+
+end

data/test/speedtest.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+#require 'rubygems'
+require 'Grok'
+#require 'ruby-prof'
+require 'pp'
+
+#RubyProf.start  
+
+iterations = 200
+pattern = "[A-z0-9_-]*\\[[0-9]+\\]"
+
+grok = Grok.new
+grok.add_pattern("FOO", pattern)
+grok.compile("%{FOO}")
+
+rubyre = Regexp.new("(?<foo>#{pattern})")
+#rubyre = Regexp.new(pattern)
+
+matches = { :grok => 0, :rubyre => 0 }
+def time(iterations, &block)
+  start = Time.now
+  data = File.open("/b/messages").readlines()
+  0.upto(iterations) do |i|
+    data.each do |line|
+      block.call(line)
+    end
+  end
+  return Time.now - start
+end
+
+groktime = time(iterations) do |line|
+  m = grok.match(line)
+  if m
+    matches[:grok] += 1
+    m.captures["FOO"]
+  end
+end
+
+rubyretime = time(iterations) do |line|
+  m = rubyre.match(line)
+  if m 
+    matches[:rubyre] += 1
+    m["foo"]
+  end
+end
+
+puts "Grok: #{groktime}"
+puts "rubyre: #{rubyretime}"
+puts matches.inspect
+#result = RubyProf.stop
+#printer = RubyProf::FlatPrinter.new(result)
+#printer.print(STDOUT, 0)
+
+
+pp matches

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: jls-grok
+version: !ruby/object:Gem::Version 
+  version: 0.1.2786
+platform: ruby
+authors: 
+- Jordan Sissel
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-04-12 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Grok ruby bindings - pattern match/extraction tool
+email: jls@semicomplete.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- sample.rb
+- INSTALL
+- ext/ruby_grok.c
+- ext/mkmf.log
+- ext/Makefile
+- ext/rgrok.h
+- ext/ruby_grokmatch.c
+- ext/ruby_grokmatch.h
+- ext/extconf.rb
+- ext/ruby_grok.o
+- ext/Grok.so
+- ext/ruby_grokmatch.o
+- test/GDB_COMMAND
+- test/general/basic_test.rb
+- test/general/captures_test.rb
+- test/Makefile
+- test/alltests.rb
+- test/speedtest.rb
+- test/patterns/quotedstring.rb
+- test/patterns/number.rb
+- test/patterns/ip.input
+- test/patterns/ip.rb
+- test/patterns/path.rb
+- test/patterns/month.rb
+- test/patterns/uri.rb
+- lib/grok.rb
+has_rdoc: true
+homepage: http://code.google.com/p/semicomplete/wiki/Grok
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: grok bindings for ruby
+test_files: []
+