jls-grok 0.1.2786
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/INSTALL +12 -0
- data/ext/Grok.so +0 -0
- data/ext/Makefile +157 -0
- data/ext/extconf.rb +7 -0
- data/ext/mkmf.log +54 -0
- data/ext/rgrok.h +9 -0
- data/ext/ruby_grok.c +190 -0
- data/ext/ruby_grok.o +0 -0
- data/ext/ruby_grokmatch.c +220 -0
- data/ext/ruby_grokmatch.h +14 -0
- data/ext/ruby_grokmatch.o +0 -0
- data/lib/grok.rb +1 -0
- data/sample.rb +43 -0
- data/test/GDB_COMMAND +29 -0
- data/test/Makefile +7 -0
- data/test/alltests.rb +6 -0
- data/test/general/basic_test.rb +58 -0
- data/test/general/captures_test.rb +88 -0
- data/test/patterns/ip.input +10000 -0
- data/test/patterns/ip.rb +32 -0
- data/test/patterns/month.rb +25 -0
- data/test/patterns/number.rb +70 -0
- data/test/patterns/path.rb +32 -0
- data/test/patterns/quotedstring.rb +54 -0
- data/test/patterns/uri.rb +44 -0
- data/test/speedtest.rb +56 -0
- metadata +90 -0
data/test/patterns/ip.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
#require 'rubygems'
|
2
|
+
require 'Grok'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
class IPPatternsTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@grok = Grok.new
|
8
|
+
path = "#{File.dirname(__FILE__)}/../../../patterns/base"
|
9
|
+
@grok.add_patterns_from_file(path)
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_ips
|
13
|
+
@grok.compile("%{IP}")
|
14
|
+
File.open("#{File.dirname(__FILE__)}/ip.input").each do |line|
|
15
|
+
line.chomp!
|
16
|
+
match = @grok.match(line)
|
17
|
+
assert_not_equal(false, match)
|
18
|
+
assert_equal(line, match.captures["IP"][0])
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_non_ips
|
23
|
+
@grok.compile("%{IP}")
|
24
|
+
nonips = %w{255.255.255.256 0.1.a.33 300.1.2.3 300 400.4.3.a 1.2.3.b
|
25
|
+
1..3.4.5 hello world}
|
26
|
+
nonips << "hello world"
|
27
|
+
nonips.each do |input|
|
28
|
+
match = @grok.match(input)
|
29
|
+
assert_equal(false, match)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#require 'rubygems'
|
2
|
+
require 'Grok'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
class MonthPatternsTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@grok = Grok.new
|
8
|
+
path = "#{File.dirname(__FILE__)}/../../../patterns/base"
|
9
|
+
@grok.add_patterns_from_file(path)
|
10
|
+
@grok.compile("%{MONTH}")
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_urls
|
14
|
+
months = ["Jan", "January", "Feb", "February", "Mar", "March", "Apr",
|
15
|
+
"April", "May", "Jun", "June", "Jul", "July", "Aug", "August",
|
16
|
+
"Sep", "September", "Oct", "October", "Nov", "November", "Dec",
|
17
|
+
"December"]
|
18
|
+
months.each do |month|
|
19
|
+
match = @grok.match(month)
|
20
|
+
assert_not_equal(false, match)
|
21
|
+
assert_equal(month, match.captures["MONTH"][0])
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
#require 'rubygems'
|
2
|
+
require 'Grok'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
class NumberPatternsTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@grok = Grok.new
|
8
|
+
path = "#{File.dirname(__FILE__)}/../../../patterns/base"
|
9
|
+
@grok.add_patterns_from_file(path)
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_match_number
|
13
|
+
@grok.compile("%{NUMBER}")
|
14
|
+
# step of a prime number near 100 so we get about 2000 iterations
|
15
|
+
#puts @grok.expanded_pattern.inspect
|
16
|
+
-100000.step(100000, 97) do |value|
|
17
|
+
match = @grok.match(value.to_s)
|
18
|
+
assert_not_equal(false, match, "#{value} should not match false")
|
19
|
+
assert_equal(value.to_s, match.captures["NUMBER"][0])
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_match_number_float
|
24
|
+
# generate some random floating point values
|
25
|
+
# always seed with the same random number, so the test is always the same
|
26
|
+
srand(0)
|
27
|
+
@grok.compile("%{NUMBER}")
|
28
|
+
0.upto(1000) do |value|
|
29
|
+
value = (rand * 100000 - 50000).to_s
|
30
|
+
match = @grok.match(value)
|
31
|
+
assert_not_equal(false, match)
|
32
|
+
assert_equal(value, match.captures["NUMBER"][0])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_match_number_amid_things
|
37
|
+
@grok.compile("%{NUMBER}")
|
38
|
+
value = "hello 12345 world"
|
39
|
+
match = @grok.match(value)
|
40
|
+
assert_not_equal(false, match)
|
41
|
+
assert_equal("12345", match.captures["NUMBER"][0])
|
42
|
+
|
43
|
+
value = "Something costs $55.4!"
|
44
|
+
match = @grok.match(value)
|
45
|
+
assert_not_equal(false, match)
|
46
|
+
assert_equal("55.4", match.captures["NUMBER"][0])
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_no_match_number
|
50
|
+
@grok.compile("%{NUMBER}")
|
51
|
+
["foo", "", " ", ".", "hello world", "-abcd"].each do |value|
|
52
|
+
match = @grok.match(value.to_s)
|
53
|
+
assert_equal(false, match)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_match_base16num
|
58
|
+
@grok.compile("%{BASE16NUM}")
|
59
|
+
# Ruby represents negative values in a strange way, so only
|
60
|
+
# test positive numbers for now.
|
61
|
+
# I don't think anyone uses negative values in hex anyway...
|
62
|
+
0.upto(1000) do |value|
|
63
|
+
[("%x" % value), ("0x%08x" % value), ("%016x" % value)].each do |hexstr|
|
64
|
+
match = @grok.match(hexstr)
|
65
|
+
assert_not_equal(false, match)
|
66
|
+
assert_equal(hexstr, match.captures["BASE16NUM"][0])
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
#require 'rubygems'
|
2
|
+
require 'Grok'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
class PathPatternsTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@grok = Grok.new
|
8
|
+
path = "#{File.dirname(__FILE__)}/../../../patterns/base"
|
9
|
+
@grok.add_patterns_from_file(path)
|
10
|
+
@grok.compile("%{PATH}")
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_unix_paths
|
14
|
+
paths = %w{/ /usr /usr/bin /usr/bin/foo /etc/motd /home/.test
|
15
|
+
/foo/bar//baz //testing /.test /%foo% /asdf/asdf,v}
|
16
|
+
paths.each do |path|
|
17
|
+
match = @grok.match(path)
|
18
|
+
assert_not_equal(false, match)
|
19
|
+
assert_equal(path, match.captures["PATH"][0])
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def test_windows_paths
|
24
|
+
paths = %w{C:\WINDOWS \\Foo\bar}
|
25
|
+
paths << "C:\\Documents and Settings\\"
|
26
|
+
paths.each do |path|
|
27
|
+
match = @grok.match(path)
|
28
|
+
assert_not_equal(false, match)
|
29
|
+
assert_equal(path, match.captures["PATH"][0])
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
#require 'rubygems'
|
2
|
+
require 'Grok'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
class QuotedStringPatternsTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@grok = Grok.new
|
8
|
+
path = "#{File.dirname(__FILE__)}/../../../patterns/base"
|
9
|
+
@grok.add_patterns_from_file(path)
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_quoted_string_common
|
13
|
+
@grok.compile("%{QUOTEDSTRING}")
|
14
|
+
inputs = ["hello", ""]
|
15
|
+
quotes = %w{" ' `}
|
16
|
+
inputs.each do |value|
|
17
|
+
quotes.each do |quote|
|
18
|
+
str = "#{quote}#{value}#{quote}"
|
19
|
+
match = @grok.match(str)
|
20
|
+
assert_not_equal(false, match)
|
21
|
+
assert_equal(str, match.captures["QUOTEDSTRING"][0])
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_quoted_string_inside_escape
|
27
|
+
@grok.compile("%{QUOTEDSTRING}")
|
28
|
+
quotes = %w{" ' `}
|
29
|
+
quotes.each do |quote|
|
30
|
+
str = "#{quote}hello \\#{quote}world\\#{quote}#{quote}"
|
31
|
+
match = @grok.match(str)
|
32
|
+
assert_not_equal(false, match)
|
33
|
+
assert_equal(str, match.captures["QUOTEDSTRING"][0])
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_escaped_quotes_no_match_quoted_string
|
38
|
+
@grok.compile("%{QUOTEDSTRING}")
|
39
|
+
inputs = ["\\\"testing\\\"", "\\\'testing\\\'", "\\\`testing\\\`",]
|
40
|
+
inputs.each do |value|
|
41
|
+
match = @grok.match(value)
|
42
|
+
assert_equal(false, match)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def test_non_quoted_strings_no_match
|
47
|
+
@grok.compile("%{QUOTEDSTRING}")
|
48
|
+
inputs = ["\\\"testing", "testing", "hello world ' something ` foo"]
|
49
|
+
inputs.each do |value|
|
50
|
+
match = @grok.match(value)
|
51
|
+
assert_equal(false, match)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
#require 'rubygems'
|
2
|
+
require 'Grok'
|
3
|
+
require 'test/unit'
|
4
|
+
|
5
|
+
class URIPatternsTest < Test::Unit::TestCase
|
6
|
+
def setup
|
7
|
+
@grok = Grok.new
|
8
|
+
path = "#{File.dirname(__FILE__)}/../../../patterns/base"
|
9
|
+
@grok.add_patterns_from_file(path)
|
10
|
+
@grok.compile("%{URI}")
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_urls
|
14
|
+
urls = ["http://www.google.com", "telnet://helloworld",
|
15
|
+
"http://www.example.com/", "http://www.example.com/test.html",
|
16
|
+
"http://www.example.com/test.html?foo=bar",
|
17
|
+
"http://www.example.com/test.html?foo=bar&fizzle=baz",
|
18
|
+
"http://www.example.com:80/test.html?foo=bar&fizzle=baz",
|
19
|
+
"https://www.example.com:443/test.html?foo=bar&fizzle=baz",
|
20
|
+
"https://user@www.example.com:443/test.html?foo=bar&fizzle=baz",
|
21
|
+
"https://user:pass@somehost/fetch.pl",
|
22
|
+
"puppet:///",
|
23
|
+
"http://www.foo.com",
|
24
|
+
"http://www.foo.com/",
|
25
|
+
"http://www.foo.com/?testing",
|
26
|
+
"http://www.foo.com/?one=two",
|
27
|
+
"http://www.foo.com/?one=two&foo=bar",
|
28
|
+
"foo://somehost.com:12345",
|
29
|
+
"foo://user@somehost.com:12345",
|
30
|
+
"foo://user@somehost.com:12345/",
|
31
|
+
"foo://user@somehost.com:12345/foo.bar/baz/fizz",
|
32
|
+
"foo://user@somehost.com:12345/foo.bar/baz/fizz?test",
|
33
|
+
"foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4",
|
34
|
+
"http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search"
|
35
|
+
]
|
36
|
+
|
37
|
+
urls.each do |url|
|
38
|
+
match = @grok.match(url)
|
39
|
+
assert_not_equal(false, match)
|
40
|
+
assert_equal(url, match.captures["URI"][0])
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
data/test/speedtest.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
#require 'rubygems'
|
4
|
+
require 'Grok'
|
5
|
+
#require 'ruby-prof'
|
6
|
+
require 'pp'
|
7
|
+
|
8
|
+
#RubyProf.start
|
9
|
+
|
10
|
+
iterations = 200
|
11
|
+
pattern = "[A-z0-9_-]*\\[[0-9]+\\]"
|
12
|
+
|
13
|
+
grok = Grok.new
|
14
|
+
grok.add_pattern("FOO", pattern)
|
15
|
+
grok.compile("%{FOO}")
|
16
|
+
|
17
|
+
rubyre = Regexp.new("(?<foo>#{pattern})")
|
18
|
+
#rubyre = Regexp.new(pattern)
|
19
|
+
|
20
|
+
matches = { :grok => 0, :rubyre => 0 }
|
21
|
+
def time(iterations, &block)
|
22
|
+
start = Time.now
|
23
|
+
data = File.open("/b/messages").readlines()
|
24
|
+
0.upto(iterations) do |i|
|
25
|
+
data.each do |line|
|
26
|
+
block.call(line)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
return Time.now - start
|
30
|
+
end
|
31
|
+
|
32
|
+
groktime = time(iterations) do |line|
|
33
|
+
m = grok.match(line)
|
34
|
+
if m
|
35
|
+
matches[:grok] += 1
|
36
|
+
m.captures["FOO"]
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
rubyretime = time(iterations) do |line|
|
41
|
+
m = rubyre.match(line)
|
42
|
+
if m
|
43
|
+
matches[:rubyre] += 1
|
44
|
+
m["foo"]
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
puts "Grok: #{groktime}"
|
49
|
+
puts "rubyre: #{rubyretime}"
|
50
|
+
puts matches.inspect
|
51
|
+
#result = RubyProf.stop
|
52
|
+
#printer = RubyProf::FlatPrinter.new(result)
|
53
|
+
#printer.print(STDOUT, 0)
|
54
|
+
|
55
|
+
|
56
|
+
pp matches
|
metadata
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: jls-grok
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.2786
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jordan Sissel
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2010-04-12 00:00:00 -07:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: eventmachine
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0"
|
24
|
+
version:
|
25
|
+
description: Grok ruby bindings - pattern match/extraction tool
|
26
|
+
email: jls@semicomplete.com
|
27
|
+
executables: []
|
28
|
+
|
29
|
+
extensions:
|
30
|
+
- ext/extconf.rb
|
31
|
+
extra_rdoc_files: []
|
32
|
+
|
33
|
+
files:
|
34
|
+
- sample.rb
|
35
|
+
- INSTALL
|
36
|
+
- ext/ruby_grok.c
|
37
|
+
- ext/mkmf.log
|
38
|
+
- ext/Makefile
|
39
|
+
- ext/rgrok.h
|
40
|
+
- ext/ruby_grokmatch.c
|
41
|
+
- ext/ruby_grokmatch.h
|
42
|
+
- ext/extconf.rb
|
43
|
+
- ext/ruby_grok.o
|
44
|
+
- ext/Grok.so
|
45
|
+
- ext/ruby_grokmatch.o
|
46
|
+
- test/GDB_COMMAND
|
47
|
+
- test/general/basic_test.rb
|
48
|
+
- test/general/captures_test.rb
|
49
|
+
- test/Makefile
|
50
|
+
- test/alltests.rb
|
51
|
+
- test/speedtest.rb
|
52
|
+
- test/patterns/quotedstring.rb
|
53
|
+
- test/patterns/number.rb
|
54
|
+
- test/patterns/ip.input
|
55
|
+
- test/patterns/ip.rb
|
56
|
+
- test/patterns/path.rb
|
57
|
+
- test/patterns/month.rb
|
58
|
+
- test/patterns/uri.rb
|
59
|
+
- lib/grok.rb
|
60
|
+
has_rdoc: true
|
61
|
+
homepage: http://code.google.com/p/semicomplete/wiki/Grok
|
62
|
+
licenses: []
|
63
|
+
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options: []
|
66
|
+
|
67
|
+
require_paths:
|
68
|
+
- lib
|
69
|
+
- ext
|
70
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: "0"
|
75
|
+
version:
|
76
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - ">="
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: "0"
|
81
|
+
version:
|
82
|
+
requirements: []
|
83
|
+
|
84
|
+
rubyforge_project:
|
85
|
+
rubygems_version: 1.3.5
|
86
|
+
signing_key:
|
87
|
+
specification_version: 3
|
88
|
+
summary: grok bindings for ruby
|
89
|
+
test_files: []
|
90
|
+
|