jls-grok 0.1.2786

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,32 @@
1
+ #require 'rubygems'
2
+ require 'Grok'
3
+ require 'test/unit'
4
+
5
+ class IPPatternsTest < Test::Unit::TestCase
6
+ def setup
7
+ @grok = Grok.new
8
+ path = "#{File.dirname(__FILE__)}/../../../patterns/base"
9
+ @grok.add_patterns_from_file(path)
10
+ end
11
+
12
+ def test_ips
13
+ @grok.compile("%{IP}")
14
+ File.open("#{File.dirname(__FILE__)}/ip.input").each do |line|
15
+ line.chomp!
16
+ match = @grok.match(line)
17
+ assert_not_equal(false, match)
18
+ assert_equal(line, match.captures["IP"][0])
19
+ end
20
+ end
21
+
22
+ def test_non_ips
23
+ @grok.compile("%{IP}")
24
+ nonips = %w{255.255.255.256 0.1.a.33 300.1.2.3 300 400.4.3.a 1.2.3.b
25
+ 1..3.4.5 hello world}
26
+ nonips << "hello world"
27
+ nonips.each do |input|
28
+ match = @grok.match(input)
29
+ assert_equal(false, match)
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,25 @@
1
+ #require 'rubygems'
2
+ require 'Grok'
3
+ require 'test/unit'
4
+
5
+ class MonthPatternsTest < Test::Unit::TestCase
6
+ def setup
7
+ @grok = Grok.new
8
+ path = "#{File.dirname(__FILE__)}/../../../patterns/base"
9
+ @grok.add_patterns_from_file(path)
10
+ @grok.compile("%{MONTH}")
11
+ end
12
+
13
+ def test_urls
14
+ months = ["Jan", "January", "Feb", "February", "Mar", "March", "Apr",
15
+ "April", "May", "Jun", "June", "Jul", "July", "Aug", "August",
16
+ "Sep", "September", "Oct", "October", "Nov", "November", "Dec",
17
+ "December"]
18
+ months.each do |month|
19
+ match = @grok.match(month)
20
+ assert_not_equal(false, match)
21
+ assert_equal(month, match.captures["MONTH"][0])
22
+ end
23
+ end
24
+
25
+ end
@@ -0,0 +1,70 @@
1
+ #require 'rubygems'
2
+ require 'Grok'
3
+ require 'test/unit'
4
+
5
+ class NumberPatternsTest < Test::Unit::TestCase
6
+ def setup
7
+ @grok = Grok.new
8
+ path = "#{File.dirname(__FILE__)}/../../../patterns/base"
9
+ @grok.add_patterns_from_file(path)
10
+ end
11
+
12
+ def test_match_number
13
+ @grok.compile("%{NUMBER}")
14
+ # step of a prime number near 100 so we get about 2000 iterations
15
+ #puts @grok.expanded_pattern.inspect
16
+ -100000.step(100000, 97) do |value|
17
+ match = @grok.match(value.to_s)
18
+ assert_not_equal(false, match, "#{value} should not match false")
19
+ assert_equal(value.to_s, match.captures["NUMBER"][0])
20
+ end
21
+ end
22
+
23
+ def test_match_number_float
24
+ # generate some random floating point values
25
+ # always seed with the same random number, so the test is always the same
26
+ srand(0)
27
+ @grok.compile("%{NUMBER}")
28
+ 0.upto(1000) do |value|
29
+ value = (rand * 100000 - 50000).to_s
30
+ match = @grok.match(value)
31
+ assert_not_equal(false, match)
32
+ assert_equal(value, match.captures["NUMBER"][0])
33
+ end
34
+ end
35
+
36
+ def test_match_number_amid_things
37
+ @grok.compile("%{NUMBER}")
38
+ value = "hello 12345 world"
39
+ match = @grok.match(value)
40
+ assert_not_equal(false, match)
41
+ assert_equal("12345", match.captures["NUMBER"][0])
42
+
43
+ value = "Something costs $55.4!"
44
+ match = @grok.match(value)
45
+ assert_not_equal(false, match)
46
+ assert_equal("55.4", match.captures["NUMBER"][0])
47
+ end
48
+
49
+ def test_no_match_number
50
+ @grok.compile("%{NUMBER}")
51
+ ["foo", "", " ", ".", "hello world", "-abcd"].each do |value|
52
+ match = @grok.match(value.to_s)
53
+ assert_equal(false, match)
54
+ end
55
+ end
56
+
57
+ def test_match_base16num
58
+ @grok.compile("%{BASE16NUM}")
59
+ # Ruby represents negative values in a strange way, so only
60
+ # test positive numbers for now.
61
+ # I don't think anyone uses negative values in hex anyway...
62
+ 0.upto(1000) do |value|
63
+ [("%x" % value), ("0x%08x" % value), ("%016x" % value)].each do |hexstr|
64
+ match = @grok.match(hexstr)
65
+ assert_not_equal(false, match)
66
+ assert_equal(hexstr, match.captures["BASE16NUM"][0])
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,32 @@
1
+ #require 'rubygems'
2
+ require 'Grok'
3
+ require 'test/unit'
4
+
5
+ class PathPatternsTest < Test::Unit::TestCase
6
+ def setup
7
+ @grok = Grok.new
8
+ path = "#{File.dirname(__FILE__)}/../../../patterns/base"
9
+ @grok.add_patterns_from_file(path)
10
+ @grok.compile("%{PATH}")
11
+ end
12
+
13
+ def test_unix_paths
14
+ paths = %w{/ /usr /usr/bin /usr/bin/foo /etc/motd /home/.test
15
+ /foo/bar//baz //testing /.test /%foo% /asdf/asdf,v}
16
+ paths.each do |path|
17
+ match = @grok.match(path)
18
+ assert_not_equal(false, match)
19
+ assert_equal(path, match.captures["PATH"][0])
20
+ end
21
+ end
22
+
23
+ def test_windows_paths
24
+ paths = %w{C:\WINDOWS \\Foo\bar}
25
+ paths << "C:\\Documents and Settings\\"
26
+ paths.each do |path|
27
+ match = @grok.match(path)
28
+ assert_not_equal(false, match)
29
+ assert_equal(path, match.captures["PATH"][0])
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,54 @@
1
+ #require 'rubygems'
2
+ require 'Grok'
3
+ require 'test/unit'
4
+
5
+ class QuotedStringPatternsTest < Test::Unit::TestCase
6
+ def setup
7
+ @grok = Grok.new
8
+ path = "#{File.dirname(__FILE__)}/../../../patterns/base"
9
+ @grok.add_patterns_from_file(path)
10
+ end
11
+
12
+ def test_quoted_string_common
13
+ @grok.compile("%{QUOTEDSTRING}")
14
+ inputs = ["hello", ""]
15
+ quotes = %w{" ' `}
16
+ inputs.each do |value|
17
+ quotes.each do |quote|
18
+ str = "#{quote}#{value}#{quote}"
19
+ match = @grok.match(str)
20
+ assert_not_equal(false, match)
21
+ assert_equal(str, match.captures["QUOTEDSTRING"][0])
22
+ end
23
+ end
24
+ end
25
+
26
+ def test_quoted_string_inside_escape
27
+ @grok.compile("%{QUOTEDSTRING}")
28
+ quotes = %w{" ' `}
29
+ quotes.each do |quote|
30
+ str = "#{quote}hello \\#{quote}world\\#{quote}#{quote}"
31
+ match = @grok.match(str)
32
+ assert_not_equal(false, match)
33
+ assert_equal(str, match.captures["QUOTEDSTRING"][0])
34
+ end
35
+ end
36
+
37
+ def test_escaped_quotes_no_match_quoted_string
38
+ @grok.compile("%{QUOTEDSTRING}")
39
+ inputs = ["\\\"testing\\\"", "\\\'testing\\\'", "\\\`testing\\\`",]
40
+ inputs.each do |value|
41
+ match = @grok.match(value)
42
+ assert_equal(false, match)
43
+ end
44
+ end
45
+
46
+ def test_non_quoted_strings_no_match
47
+ @grok.compile("%{QUOTEDSTRING}")
48
+ inputs = ["\\\"testing", "testing", "hello world ' something ` foo"]
49
+ inputs.each do |value|
50
+ match = @grok.match(value)
51
+ assert_equal(false, match)
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,44 @@
1
+ #require 'rubygems'
2
+ require 'Grok'
3
+ require 'test/unit'
4
+
5
+ class URIPatternsTest < Test::Unit::TestCase
6
+ def setup
7
+ @grok = Grok.new
8
+ path = "#{File.dirname(__FILE__)}/../../../patterns/base"
9
+ @grok.add_patterns_from_file(path)
10
+ @grok.compile("%{URI}")
11
+ end
12
+
13
+ def test_urls
14
+ urls = ["http://www.google.com", "telnet://helloworld",
15
+ "http://www.example.com/", "http://www.example.com/test.html",
16
+ "http://www.example.com/test.html?foo=bar",
17
+ "http://www.example.com/test.html?foo=bar&fizzle=baz",
18
+ "http://www.example.com:80/test.html?foo=bar&fizzle=baz",
19
+ "https://www.example.com:443/test.html?foo=bar&fizzle=baz",
20
+ "https://user@www.example.com:443/test.html?foo=bar&fizzle=baz",
21
+ "https://user:pass@somehost/fetch.pl",
22
+ "puppet:///",
23
+ "http://www.foo.com",
24
+ "http://www.foo.com/",
25
+ "http://www.foo.com/?testing",
26
+ "http://www.foo.com/?one=two",
27
+ "http://www.foo.com/?one=two&foo=bar",
28
+ "foo://somehost.com:12345",
29
+ "foo://user@somehost.com:12345",
30
+ "foo://user@somehost.com:12345/",
31
+ "foo://user@somehost.com:12345/foo.bar/baz/fizz",
32
+ "foo://user@somehost.com:12345/foo.bar/baz/fizz?test",
33
+ "foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4",
34
+ "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search"
35
+ ]
36
+
37
+ urls.each do |url|
38
+ match = @grok.match(url)
39
+ assert_not_equal(false, match)
40
+ assert_equal(url, match.captures["URI"][0])
41
+ end
42
+ end
43
+
44
+ end
data/test/speedtest.rb ADDED
@@ -0,0 +1,56 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #require 'rubygems'
4
+ require 'Grok'
5
+ #require 'ruby-prof'
6
+ require 'pp'
7
+
8
+ #RubyProf.start
9
+
10
+ iterations = 200
11
+ pattern = "[A-z0-9_-]*\\[[0-9]+\\]"
12
+
13
+ grok = Grok.new
14
+ grok.add_pattern("FOO", pattern)
15
+ grok.compile("%{FOO}")
16
+
17
+ rubyre = Regexp.new("(?<foo>#{pattern})")
18
+ #rubyre = Regexp.new(pattern)
19
+
20
+ matches = { :grok => 0, :rubyre => 0 }
21
+ def time(iterations, &block)
22
+ start = Time.now
23
+ data = File.open("/b/messages").readlines()
24
+ 0.upto(iterations) do |i|
25
+ data.each do |line|
26
+ block.call(line)
27
+ end
28
+ end
29
+ return Time.now - start
30
+ end
31
+
32
+ groktime = time(iterations) do |line|
33
+ m = grok.match(line)
34
+ if m
35
+ matches[:grok] += 1
36
+ m.captures["FOO"]
37
+ end
38
+ end
39
+
40
+ rubyretime = time(iterations) do |line|
41
+ m = rubyre.match(line)
42
+ if m
43
+ matches[:rubyre] += 1
44
+ m["foo"]
45
+ end
46
+ end
47
+
48
+ puts "Grok: #{groktime}"
49
+ puts "rubyre: #{rubyretime}"
50
+ puts matches.inspect
51
+ #result = RubyProf.stop
52
+ #printer = RubyProf::FlatPrinter.new(result)
53
+ #printer.print(STDOUT, 0)
54
+
55
+
56
+ pp matches
metadata ADDED
@@ -0,0 +1,90 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: jls-grok
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.2786
5
+ platform: ruby
6
+ authors:
7
+ - Jordan Sissel
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+
12
+ date: 2010-04-12 00:00:00 -07:00
13
+ default_executable:
14
+ dependencies:
15
+ - !ruby/object:Gem::Dependency
16
+ name: eventmachine
17
+ type: :runtime
18
+ version_requirement:
19
+ version_requirements: !ruby/object:Gem::Requirement
20
+ requirements:
21
+ - - ">="
22
+ - !ruby/object:Gem::Version
23
+ version: "0"
24
+ version:
25
+ description: Grok ruby bindings - pattern match/extraction tool
26
+ email: jls@semicomplete.com
27
+ executables: []
28
+
29
+ extensions:
30
+ - ext/extconf.rb
31
+ extra_rdoc_files: []
32
+
33
+ files:
34
+ - sample.rb
35
+ - INSTALL
36
+ - ext/ruby_grok.c
37
+ - ext/mkmf.log
38
+ - ext/Makefile
39
+ - ext/rgrok.h
40
+ - ext/ruby_grokmatch.c
41
+ - ext/ruby_grokmatch.h
42
+ - ext/extconf.rb
43
+ - ext/ruby_grok.o
44
+ - ext/Grok.so
45
+ - ext/ruby_grokmatch.o
46
+ - test/GDB_COMMAND
47
+ - test/general/basic_test.rb
48
+ - test/general/captures_test.rb
49
+ - test/Makefile
50
+ - test/alltests.rb
51
+ - test/speedtest.rb
52
+ - test/patterns/quotedstring.rb
53
+ - test/patterns/number.rb
54
+ - test/patterns/ip.input
55
+ - test/patterns/ip.rb
56
+ - test/patterns/path.rb
57
+ - test/patterns/month.rb
58
+ - test/patterns/uri.rb
59
+ - lib/grok.rb
60
+ has_rdoc: true
61
+ homepage: http://code.google.com/p/semicomplete/wiki/Grok
62
+ licenses: []
63
+
64
+ post_install_message:
65
+ rdoc_options: []
66
+
67
+ require_paths:
68
+ - lib
69
+ - ext
70
+ required_ruby_version: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: "0"
75
+ version:
76
+ required_rubygems_version: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - ">="
79
+ - !ruby/object:Gem::Version
80
+ version: "0"
81
+ version:
82
+ requirements: []
83
+
84
+ rubyforge_project:
85
+ rubygems_version: 1.3.5
86
+ signing_key:
87
+ specification_version: 3
88
+ summary: grok bindings for ruby
89
+ test_files: []
90
+