jls-grok 0.4.7 → 0.5.2

data/lib/Grok.rb

@@ -0,0 +1,3 @@
+require "grok.rb"
+
+# compat for when grok was Grok.so

data/lib/grok-pure.rb

@@ -0,0 +1,137 @@
+require "rubygems"
+
+# TODO(sissel): Check if 'grok' c-ext has been loaded and abort?
+class Grok
+  attr_accessor :pattern
+  attr_accessor :expanded_pattern
+  
+  PATTERN_RE = \
+    /%{    # match '%{' not prefixed with '\'
+       (?<name>     # match the pattern name
+         (?<pattern>[A-z0-9]+)
+         (?::(?<subname>[A-z0-9_:]+))?
+       )
+       (?:=(?<definition>
+         (?:
+           (?:[^{}\\]+|\\.+)+
+           |
+           (?<curly>\{(?:(?>[^{}]+|(?>\\[{}])+)|(\g<curly>))*\})+
+         )+
+       ))?
+       [^}]*
+     }/x
+
+  GROK_OK = 0
+  GROK_ERROR_FILE_NOT_ACCESSIBLE = 1
+  GROK_ERROR_PATTERN_NOT_FOUND = 2
+  GROK_ERROR_UNEXPECTED_READ_SIZE = 3
+  GROK_ERROR_COMPILE_FAILED = 4
+  GROK_ERROR_UNINITIALIZED = 5
+  GROK_ERROR_PCRE_ERROR = 6
+  GROK_ERROR_NOMATCH = 7
+
+  public
+  def initialize
+    @patterns = {}
+
+    # TODO(sissel): Throw exception if we aren't using Ruby 1.9.2 or newer.
+  end # def initialize
+
+  public
+  def add_pattern(name, pattern)
+    #puts "#{name} => #{pattern}"
+    @patterns[name] = pattern
+    return nil
+  end
+
+  public
+  def add_patterns_from_file(path)
+    file = File.new(path, "r")
+    file.each do |line|
+      next if line =~ /^\s*#/
+      #puts "Pattern: #{line}"
+      name, pattern = line.gsub(/^\s*/, "").split(/\s+/, 2)
+      next if pattern.nil?
+      add_pattern(name, pattern.chomp)
+    end
+    return nil
+  end # def add_patterns_from_file
+
+  public
+  def compile(pattern)
+    @capture_map = {}
+
+    iterations_left = 100
+    @pattern = pattern
+    @expanded_pattern = pattern
+    index = 0
+
+    # Replace any instances of '%{FOO}' with that pattern.
+    loop do
+      if iterations_left == 0
+        raise "Deep recursion pattern compilation of #{pattern.inspect} - expanded: #{@expanded_pattern.inspect}"
+      end
+      iterations_left -= 1
+      m = PATTERN_RE.match(@expanded_pattern)
+      break if !m
+
+      if m["definition"]
+        add_pattern(m["pattern"], m["definition"])
+      end
+
+      if @patterns.include?(m["pattern"])
+        # create a named capture index that we can push later as the named
+        # pattern. We do this because ruby regexp can't capture something
+        # by the same name twice.
+        p = @patterns[m["pattern"]]
+
+        capture = "a#{index}" # named captures have to start with letters?
+        #capture = "%04d" % "#{index}" # named captures have to start with letters?
+        replacement_pattern = "(?<#{capture}>#{p})"
+        #p(:input => m[0], :pattern => replacement_pattern)
+        @capture_map[capture] = m["name"]
+        @expanded_pattern.sub!(m[0], replacement_pattern)
+        index += 1
+      end
+    end
+
+    @regexp = Regexp.new(@expanded_pattern)
+  end # def compile
+
+  public
+  def match(text)
+    match = @regexp.match(text)
+
+    if match
+      grokmatch = Grok::Match.new
+      grokmatch.subject = text
+      grokmatch.start, grokmatch.end = match.offset(0)
+      grokmatch.grok = self
+      grokmatch.match = match
+      return grokmatch
+    else
+      return false
+    end
+  end # def match
+
+  public
+  def discover(input)
+    init_discover if @discover == nil
+
+    return @discover.discover(input)
+  end # def discover
+
+  private
+  def init_discover
+    @discover = GrokDiscover.new(self)
+    @discover.logmask = logmask
+  end # def init_discover
+
+  public
+  def capture_name(id)
+    return @capture_map[id]
+  end # def capture_name
+end # Grok
+
+require "grok/pure/match"
+require "grok/pure/pile"

data/lib/grok.rb

@@ -1,6 +1,7 @@
 require "rubygems"
 require "ffi"
 
+# TODO(sissel): Check if 'grok-pure' has been loaded and abort?
 class Grok < FFI::Struct
   module CGrok
     extend FFI::Library
@@ -128,5 +129,5 @@ class Grok < FFI::Struct
   end
 end # Grok
 
-require "grok/match"
-require "grok/pile"
+require "grok/c-ext/match"
+require "grok/c-ext/pile"

data/lib/grok/pure/match.rb

@@ -0,0 +1,45 @@
+require "grok-pure"
+
+class Grok::Match
+  attr_accessor :subject
+  attr_accessor :start
+  attr_accessor :end
+  attr_accessor :grok
+  attr_accessor :match
+
+  public
+  def initialize
+    @captures = nil
+  end
+
+  public
+  def each_capture
+    @captures = Hash.new { |h, k| h[k] = Array.new }
+
+    #p :expanded => @grok.expanded_pattern
+    #p :map => @grok.capture_map
+    @match.names.zip(@match.captures).each do |id, value|
+      #p :match => id, :value => value
+      name = @grok.capture_name(id)
+      #next if value == nil
+      yield name, value
+    end
+
+  end # def each_capture
+
+  public
+  def captures
+    if @captures.nil?
+      @captures = Hash.new { |h,k| h[k] = [] }
+      each_capture do |key, val|
+        @captures[key] << val
+      end
+    end
+    return @captures
+  end # def captures
+
+  public
+  def [](name)
+    return captures[name]
+  end # def []
+end # Grok::Match

data/lib/grok/pure/pile.rb

@@ -0,0 +1,56 @@
+require "grok-pure"
+
+# A grok pile is an easy way to have multiple patterns together so
+# that you can try to match against each one.
+# The API provided should be similar to the normal Grok
+# interface, but you can compile multiple patterns and match will
+# try each one until a match is found.
+class Grok
+  class Pile
+    def initialize
+      @groks = []
+      @patterns = {}
+      @pattern_files = []
+    end # def initialize
+
+    # see Grok#add_pattern
+    def add_pattern(name, string)
+      @patterns[name] = string
+    end # def add_pattern
+
+    # see Grok#add_patterns_from_file
+    def add_patterns_from_file(path)
+      if !File.exists?(path)
+        raise "File does not exist: #{path}"
+      end
+      @pattern_files << path
+    end # def add_patterns_from_file
+
+    # see Grok#compile
+    def compile(pattern)
+      grok = Grok.new
+      @patterns.each do |name, value|
+        grok.add_pattern(name, value)
+      end
+      @pattern_files.each do |path|
+        grok.add_patterns_from_file(path)
+      end
+      grok.compile(pattern)
+      @groks << grok
+    end # def compile
+
+    # Slight difference from Grok#match in that it returns
+    # the Grok instance that matched successfully in addition
+    # to the GrokMatch result.
+    # See also: Grok#match
+    def match(string)
+      @groks.each do |grok|
+        match = grok.match(string)
+        if match
+          return [grok, match]
+        end
+      end
+      return false
+    end # def match
+  end # class Pile
+end # class Grok

metadata

@@ -2,7 +2,7 @@
 name: jls-grok
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.4.7
+  version: 0.5.2
 platform: ruby
 authors: 
   - Jordan Sissel
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-06 00:00:00 -07:00
+date: 2011-08-21 00:00:00 -07:00
 default_executable: 
 dependencies: 
   - !ruby/object:Gem::Dependency 
@@ -20,7 +20,7 @@ dependencies:
     requirement: &id001 !ruby/object:Gem::Requirement 
       none: false
       requirements: 
-        - - ">="
+        - - ~>
           - !ruby/object:Gem::Version 
             version: 0.6.3
     type: :runtime
@@ -36,32 +36,13 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-  - INSTALL
-  - Rakefile
-  - examples/grok-web.rb
-  - examples/pattern-discovery.rb
-  - examples/test.rb
-  - grok.gemspec
+  - lib/Grok.rb
+  - lib/grok-pure.rb
   - lib/grok.rb
-  - lib/grok/match.rb
-  - lib/grok/pile.rb
-  - test/Makefile
-  - test/alltests.rb
-  - test/general/basic_test.rb
-  - test/general/captures_test.rb
-  - test/patterns/day.rb
-  - test/patterns/host.rb
-  - test/patterns/ip.input
-  - test/patterns/ip.rb
-  - test/patterns/iso8601.rb
-  - test/patterns/month.rb
-  - test/patterns/number.rb
-  - test/patterns/path.rb
-  - test/patterns/prog.rb
-  - test/patterns/quotedstring.rb
-  - test/patterns/uri.rb
-  - test/run.sh
-  - test/speedtest.rb
+  - lib/grok/c-ext/pile.rb
+  - lib/grok/c-ext/match.rb
+  - lib/grok/pure/pile.rb
+  - lib/grok/pure/match.rb
 has_rdoc: true
 homepage: http://code.google.com/p/semicomplete/wiki/Grok
 licenses: []
@@ -72,7 +53,6 @@ rdoc_options: []
 require_paths: 
   - lib
   - lib
-  - ext
 required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 

data/INSTALL

@@ -1,12 +0,0 @@
-- You'll need grok installed.
-From 'grok', do:
-% make install
-# This will install grok, libgrok, and grok's headers
-
-- You'll need the 'Grok' ruby module installed.
-From 'grok/ruby' do:
-% ruby extconf.rb
-% make install
-
-# Test with:
-% ruby -e 'require "Grok"; puts "Grok OK"'

data/Rakefile

@@ -1,12 +0,0 @@
-task :default => [:package]
-
-task :package do
-  system("make -C ext clean; rm ext/Makefile")
-  system("svn up")
-  system("gem build grok.gemspec")
-end
-
-task :publish do
-  latest_gem = %x{ls -t jls-grok*.gem}.split("\n").first
-  system("gem push #{latest_gem}")
-end

data/examples/grok-web.rb

@@ -1,131 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Simple web application that will let you feed grok's discovery feature
-# a bunch of data, and grok will show you patterns found and the results
-# of that pattern as matched against the same input.
-
-require 'rubygems'
-require 'sinatra'
-require 'grok'
-
-get '/' do
-  redirect "/demo/grok-discover/index"
-end
-
-get "/demo/grok-discover/index" do
-  haml :index
-end
-
-post "/demo/grok-discover/grok" do
-  grok = Grok.new
-  grok.add_patterns_from_file("/usr/local/share/grok/patterns/base")
-  @results = []
-  params[:data].split("\n").each do |line|
-    pattern = grok.discover(line)
-    grok.compile(pattern)
-    match = grok.match(line)
-    puts "Got input: #{line}"
-    puts " => pattern: (#{match != false}) #{pattern}"
-    @results << { 
-        :input => line,
-        :pattern => grok.pattern.gsub(/\\Q|\\E/, ""),
-        :full_pattern => grok.expanded_pattern,
-        :match => (match and match.captures or false),
-    }
-  end
-  haml :grok
-end
-
-get "/demo/grok-discover/style.css" do
-  sass :style
-end
-
-__END__
-@@ style
-h1
-  color: red
-.original
-.regexp
-  display: block
-  border: 1px solid grey
-  padding: 1em
-
-.results
-  width: 80%
-  margin-left: auto
-  th
-    text-align: left
-  td
-    border-top: 1px solid black
-@@ layout
-%html
-  %head
-    %title Grok Web
-    %link{:rel => "stylesheet", :href => "/demo/grok-discover/style.css"}
-  %body
-    =yield
-
-@@ index
-#header
-  %h1 Grok Web
-#content
-  Paste some log data below. I'll do my best to have grok generate a pattern for you.
-
-  %p
-    Learn more about grok here:
-    %a{:href => "http://code.google.com/p/semicomplete/wiki/Grok"} Grok
-
-  %p
-    This is running off of my cable modem for now, so if it's sluggish, that's
-    why. Be gentle.
-  %form{:action => "/demo/grok-discover/grok", :method => "post"}
-    %textarea{:name => "data", :rows => 10, :cols => 80}
-    %br
-    %input{:type => "submit", :value=>"submit"}
-
-@@ grok
-#header
-  %h1 Grok Results
-  %h3
-    %a{:href => "/demo/grok-discover/index"} Try more?
-#content
-  %p
-    Below is grok's analysis of the data you provided. Each line is analyzed
-    separately. It uses grok's standard library of known patterns to give you a
-    pattern that grok can use to match more logs like the lines you provided.
-  %p
-    The results may not be perfect, but it gives you a head start on coming up with
-    log patterns for 
-    %a{:href => "http://code.google.com/p/semicomplete/wiki/Grok"} grok 
-    and 
-    %a{:href => "http://code.google.com/p/logstash/"} logstash
-  %ol
-    - @results.each do |result|
-      %li
-        %p.original
-          %b Original:
-          %br= result[:input]
-        %p 
-          %b Pattern:
-          %br
-          %span.pattern= result[:pattern]
-        %p
-          %b 
-            Generated Regular Expression
-          %small
-            %i You could have written this by hand, be glad you didn't have to.
-          %code.regexp= result[:full_pattern].gsub("<", "&lt;")
-        %p
-          If you wanted to test this, you can paste the above expression into
-          pcretest(1) and it should match your input. 
-        %p
-          %b Capture Results
-          %table.results
-            %tr
-              %th Name
-              %th Value
-            - result[:match].each do |key,val|
-              - val.each do |v|
-                %tr
-                  %td= key
-                  %td= v