jls-grok 0.1.2787 → 0.1.2821

data/ext/Makefile

@@ -87,8 +87,8 @@ extout_prefix =
 target_prefix = 
 LOCAL_LIBS = 
 LIBS = $(LIBRUBYARG_SHARED) -lgrok  -lpthread -lrt -ldl -lcrypt -lm   -lc
-SRCS = ruby_grok.c ruby_grokmatch.c
-OBJS = ruby_grok.o ruby_grokmatch.o
+SRCS = ruby_grok.c ruby_grokmatch.c ruby_grokdiscover.c
+OBJS = ruby_grok.o ruby_grokmatch.o ruby_grokdiscover.o
 TARGET = Grok
 DLLIB = $(TARGET).so
 EXTSTATIC = 

data/ext/extconf.rb

@@ -1,7 +1,6 @@
 require "mkmf"
 find_header("tcutil.h", "/usr/local/include")
 find_header("pcre.h", "/usr/local/include")
-find_header("grok.h", "../")
-find_library("grok", "grok_init", "../")
-
+find_header("grok.h", "/usr/local/include", "../../")
+find_library("grok", "grok_init", "../", "/usr/local/lib")
 create_makefile("Grok")

data/ext/rgrok.h

@@ -1,9 +1,9 @@
 #ifndef _RGROK_H_
 #define _RGROK_H_ 
 #include <ruby.h>
-
-//VALUE cGrok = Qnil;
-//VALUE cGrokMatch = Qnil;
+#include <grok.h>
 
 extern VALUE rGrok_new(VALUE klass);
+extern VALUE rGrok_new_from_grok(grok_t *grok);
+
 #endif /* _RGROK_H_ */

data/ext/ruby_grok.c

@@ -6,14 +6,16 @@
 VALUE cGrok; /* Grok class object */
 
 extern VALUE cGrokMatch;
+extern VALUE cGrokDiscover;
 extern void Init_GrokMatch();
+extern void Init_GrokDiscover();
 
 static VALUE rGrok_initialize(VALUE self) {
   /* empty */
   return Qnil;
 }
 
-static void rGrok_free(void *p) {
+void rGrok_free(void *p) {
   grok_t *grok = (grok_t *)p;
 
   /* we strdup our pattern from ruby and rb_str2cstr */
@@ -35,6 +37,13 @@ VALUE rGrok_new(VALUE klass) {
   return rgrok;
 }
 
+VALUE rGrok_new_from_grok(grok_t *grok) {
+  VALUE rgrok;
+  rgrok = Data_Wrap_Struct(cGrok, 0, rGrok_free, grok);
+  rb_obj_call_init(rgrok, 0, 0);
+  return rgrok;
+}
+
 VALUE rGrok_compile(VALUE self, VALUE pattern) {
   grok_t *grok;
   char *c_pattern = NULL;
@@ -187,4 +196,5 @@ void Init_Grok() {
   rb_define_method(cGrok, "patterns", rGrok_patterns, 0);
 
   Init_GrokMatch();
+  Init_GrokDiscover();
 }

data/ext/ruby_grokdiscover.c

@@ -0,0 +1,54 @@
+#include "rgrok.h"
+#include <grok.h>
+
+VALUE cGrokDiscover;
+extern VALUE cGrok;
+
+static void rGrokDiscover_free(void *p);
+
+VALUE rGrokDiscover_new(VALUE klass, VALUE grok) {
+  VALUE rgd;
+  grok_discover_t *gdt = ALLOC(grok_discover_t); //grok_discover_new();
+  rgd = Data_Wrap_Struct(klass, 0, rGrokDiscover_free, gdt);
+
+  VALUE initargs[1] = { grok };
+  rb_obj_call_init(rgd, 1, initargs);
+  return (VALUE)rgd;
+}
+
+static void rGrokDiscover_free(void *p) {
+  grok_discover_t *gdt = p;
+  grok_discover_free(gdt);
+}
+
+VALUE rGrokDiscover_initialize(VALUE self, VALUE rb_grok) {
+  grok_discover_t *gdt;
+  grok_t *grok;
+  Data_Get_Struct(self, grok_discover_t, gdt);
+  Data_Get_Struct(rb_grok, grok_t, grok);
+
+  grok_discover_init(gdt, grok);
+  return Qnil;
+}
+
+
+VALUE rGrokDiscover_discover(VALUE self, VALUE input) {
+  char *cstr_discovery;
+  char *cstr_input;
+  long unused_input_len;
+  int discovery_len;
+  grok_discover_t *gdt;
+  grok_t *grok;
+
+  Data_Get_Struct(self, grok_discover_t, gdt);
+  cstr_input = rb_str2cstr(input, &unused_input_len);
+  grok_discover(gdt, cstr_input, &cstr_discovery, &discovery_len);
+  return rb_str_new(cstr_discovery, discovery_len);
+}
+
+void Init_GrokDiscover() {
+  cGrokDiscover = rb_define_class("GrokDiscover", rb_cObject);
+  rb_define_singleton_method(cGrokDiscover, "new", rGrokDiscover_new, 1);
+  rb_define_method(cGrokDiscover, "initialize", rGrokDiscover_initialize, 1);
+  rb_define_method(cGrokDiscover, "discover", rGrokDiscover_discover, 1);
+}

data/lib/grok.rb

@@ -1 +1,14 @@
 require "Grok"
+
+# extend Grok to add simpler access to the discover feature.
+class Grok
+  def discover(input)
+    init_discover if @discover == nil
+
+    return @discover.discover(input)
+  end
+
+  def init_discover
+    @discover = GrokDiscover.new(self)
+  end
+end

data/test/alltests.rb

@@ -1,6 +1,8 @@
 require 'test/unit'
+$: << "../lib"
 
 Dir["#{File.dirname(__FILE__)}/*/**/*.rb"].each do |file|
+  puts "Loading tests: #{file}"
   require file
 end
 

data/test/patterns/day.rb

@@ -0,0 +1,22 @@
+require 'grok'
+require 'test/unit'
+
+class DayPatternsTest < Test::Unit::TestCase
+  def setup
+    @grok = Grok.new
+    path = "#{File.dirname(__FILE__)}/../../../patterns/base"
+    @grok.add_patterns_from_file(path)
+    @grok.compile("%{DAY}")
+  end
+
+  def test_days
+    days = %w{Mon Monday Tue Tuesday Wed Wednesday Thu Thursday Fri Friday
+                Sat Saturday Sun Sunday}
+    days.each do |day|
+      match = @grok.match(day)
+      assert_not_equal(false, day, "Expected #{day} to match.")
+      assert_equal(day, match.captures["DAY"][0])
+    end
+  end
+
+end

data/test/patterns/ip.rb

@@ -1,5 +1,4 @@
-#require 'rubygems'
-require 'Grok'
+require 'grok'
 require 'test/unit'
 
 class IPPatternsTest < Test::Unit::TestCase

data/test/patterns/month.rb

@@ -1,5 +1,4 @@
-#require 'rubygems'
-require 'Grok'
+require 'grok'
 require 'test/unit'
 
 class MonthPatternsTest < Test::Unit::TestCase
@@ -10,14 +9,14 @@ class MonthPatternsTest < Test::Unit::TestCase
     @grok.compile("%{MONTH}")
   end
 
-  def test_urls
+  def test_months
     months = ["Jan", "January", "Feb", "February", "Mar", "March", "Apr",
               "April", "May", "Jun", "June", "Jul", "July", "Aug", "August",
               "Sep", "September", "Oct", "October", "Nov", "November", "Dec",
               "December"]
     months.each do |month|
       match = @grok.match(month)
-      assert_not_equal(false, match)
+      assert_not_equal(false, match, "Expected #{month} to match")
       assert_equal(month, match.captures["MONTH"][0])
     end
   end

data/test/patterns/number.rb

@@ -1,5 +1,4 @@
-#require 'rubygems'
-require 'Grok'
+require 'grok'
 require 'test/unit'
 
 class NumberPatternsTest < Test::Unit::TestCase

data/test/patterns/path.rb

@@ -1,5 +1,4 @@
-#require 'rubygems'
-require 'Grok'
+require 'grok'
 require 'test/unit'
 
 class PathPatternsTest < Test::Unit::TestCase
@@ -21,11 +20,11 @@ class PathPatternsTest < Test::Unit::TestCase
   end
 
   def test_windows_paths
-    paths = %w{C:\WINDOWS \\Foo\bar}
+    paths = %w{C:\WINDOWS \\\\Foo\bar \\\\1.2.3.4\C$ \\\\some\path\here.exe}
     paths << "C:\\Documents and Settings\\"
     paths.each do |path|
       match = @grok.match(path)
-      assert_not_equal(false, match)
+      assert_not_equal(false, match, "Expected #{path} to match, but it didn't.")
       assert_equal(path, match.captures["PATH"][0])
     end
   end

data/test/patterns/uri.rb

@@ -1,5 +1,4 @@
-#require 'rubygems'
-require 'Grok'
+require 'grok'
 require 'test/unit'
 
 class URIPatternsTest < Test::Unit::TestCase
@@ -31,12 +30,15 @@ class URIPatternsTest < Test::Unit::TestCase
             "foo://user@somehost.com:12345/foo.bar/baz/fizz",
             "foo://user@somehost.com:12345/foo.bar/baz/fizz?test",
             "foo://user@somehost.com:12345/foo.bar/baz/fizz?test=1&sink&foo=4",
-            "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search"
+            "http://www.google.com/search?hl=en&source=hp&q=hello+world+%5E%40%23%24&btnG=Google+Search",
+            "http://www.freebsd.org/cgi/url.cgi?ports/sysutils/grok/pkg-descr",
+            "http://www.google.com/search?q=CAPTCHA+ssh&start=0&ie=utf-8&oe=utf-8&client=firefox-a&rls=org.mozilla:en-US:official",
+            "svn+ssh://somehost:12345/testing",
            ]
             
     urls.each do |url|
       match = @grok.match(url)
-      assert_not_equal(false, match)
+      assert_not_equal(false, match, "Expected this to match: #{url}")
       assert_equal(url, match.captures["URI"][0])
     end
   end

data/test/speedtest.rb

@@ -7,24 +7,24 @@ require 'pp'
 
 #RubyProf.start  
 
-iterations = 200
+iterations = 20000
 pattern = "[A-z0-9_-]*\\[[0-9]+\\]"
 
 grok = Grok.new
-grok.add_pattern("FOO", pattern)
-grok.compile("%{FOO}")
+grok.add_patterns_from_file("../../patterns/base")
+grok.compile("%{COMBINEDAPACHELOG}")
 
-rubyre = Regexp.new("(?<foo>#{pattern})")
+#rubyre = Regexp.new("(?<foo>#{pattern})")
 #rubyre = Regexp.new(pattern)
 
 matches = { :grok => 0, :rubyre => 0 }
+failures = { :grok => 0, :rubyre => 0 }
 def time(iterations, &block)
   start = Time.now
-  data = File.open("/b/messages").readlines()
-  0.upto(iterations) do |i|
-    data.each do |line|
-      block.call(line)
-    end
+  file = File.open("/b/logs/access")
+  data = (1 .. iterations).collect { file.readline() }
+  data.each do |line|
+    block.call(line)
   end
   return Time.now - start
 end
@@ -34,23 +34,24 @@ groktime = time(iterations) do |line|
   if m
     matches[:grok] += 1
     m.captures["FOO"]
+  else
+    puts line
+    failures[:grok] +=1
   end
 end
 
-rubyretime = time(iterations) do |line|
-  m = rubyre.match(line)
-  if m 
-    matches[:rubyre] += 1
-    m["foo"]
-  end
-end
-
-puts "Grok: #{groktime}"
-puts "rubyre: #{rubyretime}"
-puts matches.inspect
+#rubyretime = time(iterations) do |line|
+  #m = rubyre.match(line)
+  #if m 
+    #matches[:rubyre] += 1
+    #m["foo"]
+  #end
+#end
+
+puts "Grok: #{matches[:grok] / groktime}"
+puts failures.inspect
+#puts "rubyre: #{rubyretime}"
+#puts matches.inspect
 #result = RubyProf.stop
 #printer = RubyProf::FlatPrinter.new(result)
 #printer.print(STDOUT, 0)
-
-
-pp matches

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jls-grok
 version: !ruby/object:Gem::Version 
-  version: 0.1.2787
+  version: 0.1.2821
 platform: ruby
 authors: 
 - Jordan Sissel
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-12 00:00:00 -07:00
+date: 2010-04-16 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -25,22 +25,19 @@ files:
 - sample.rb
 - INSTALL
 - ext/ruby_grok.c
-- ext/mkmf.log
 - ext/Makefile
 - ext/rgrok.h
 - ext/ruby_grokmatch.c
 - ext/ruby_grokmatch.h
 - ext/extconf.rb
-- ext/ruby_grok.o
-- ext/Grok.so
-- ext/ruby_grokmatch.o
-- test/GDB_COMMAND
+- ext/ruby_grokdiscover.c
 - test/general/basic_test.rb
 - test/general/captures_test.rb
 - test/Makefile
 - test/alltests.rb
 - test/speedtest.rb
 - test/patterns/quotedstring.rb
+- test/patterns/day.rb
 - test/patterns/number.rb
 - test/patterns/ip.input
 - test/patterns/ip.rb

data/ext/mkmf.log

@@ -1,54 +0,0 @@
-find_header: checking for tcutil.h in /usr/local/include... -------------------- yes
-
-"gcc -E -I. -I/usr/lib/ruby/1.8/x86_64-linux -I.    -fno-strict-aliasing -g -g -O2  -fPIC    conftest.c -o conftest.i"
-checked program was:
-/* begin */
-1: #include <tcutil.h>
-/* end */
-
---------------------
-
-find_header: checking for pcre.h in /usr/local/include... -------------------- yes
-
-"gcc -E -I. -I/usr/lib/ruby/1.8/x86_64-linux -I.    -fno-strict-aliasing -g -g -O2  -fPIC    conftest.c -o conftest.i"
-checked program was:
-/* begin */
-1: #include <pcre.h>
-/* end */
-
---------------------
-
-find_header: checking for grok.h in ../... -------------------- yes
-
-"gcc -E -I. -I/usr/lib/ruby/1.8/x86_64-linux -I.    -fno-strict-aliasing -g -g -O2  -fPIC    conftest.c -o conftest.i"
-checked program was:
-/* begin */
-1: #include <grok.h>
-/* end */
-
---------------------
-
-find_library: checking for grok_init() in -lgrok... -------------------- yes
-
-"gcc -o conftest -I. -I/usr/lib/ruby/1.8/x86_64-linux -I.    -fno-strict-aliasing -g -g -O2  -fPIC   conftest.c  -L. -L/usr/lib -L. -Wl,-Bsymbolic-functions -rdynamic -Wl,-export-dynamic     -lruby1.8-static -lgrok  -lpthread -lrt -ldl -lcrypt -lm   -lc"
-conftest.c: In function 't':
-conftest.c:3: error: 'grok_init' undeclared (first use in this function)
-conftest.c:3: error: (Each undeclared identifier is reported only once
-conftest.c:3: error: for each function it appears in.)
-checked program was:
-/* begin */
-1: /*top*/
-2: int main() { return 0; }
-3: int t() { void ((*volatile p)()); p = (void ((*)()))grok_init; return 0; }
-/* end */
-
-"gcc -o conftest -I. -I/usr/lib/ruby/1.8/x86_64-linux -I.    -fno-strict-aliasing -g -g -O2  -fPIC   conftest.c  -L. -L/usr/lib -L. -Wl,-Bsymbolic-functions -rdynamic -Wl,-export-dynamic     -lruby1.8-static -lgrok  -lpthread -lrt -ldl -lcrypt -lm   -lc"
-checked program was:
-/* begin */
-1: /*top*/
-2: int main() { return 0; }
-3: int t() { grok_init(); return 0; }
-/* end */
-
---------------------
-

data/test/GDB_COMMAND

@@ -1,29 +0,0 @@
-set confirm off
-delete 
-
-break rGrokMatch_new_from_grok_match if strcmp(gm->subject, "40079") == 0
-commands
-printf "subject: %s\n", gm->subject
-delete 
-break rGrokMatch_captures
-cont
-end
-
-set confirm off
-delete 
-break rGrokMatch_new_from_grok_match if strcmp(gm->subject, "-29086") == 0
-commands
-print *gm
-delete
-break rGrokMatch_captures
-cont
-end
-
-run
-
-next
-print *gm
-print *(gm->grok)
-delete
-cont
-