jls-grok 0.9.6 → 0.10.0

data/lib/grok-pure.rb

@@ -4,9 +4,17 @@ require "cabin"
 
 # TODO(sissel): Check if 'grok' c-ext has been loaded and abort?
 class Grok
+  # The pattern input
   attr_accessor :pattern
+  
+  # The fully-expanded pattern (in regex form)
   attr_accessor :expanded_pattern
+
+  # The logger
   attr_accessor :logger
+
+  # The dictionary of pattern names to pattern expressions
+  attr_accessor :patterns
   
   PATTERN_RE = \
     /%\{    # match '%{' not prefixed with '\'
@@ -58,6 +66,7 @@ class Grok
       next if line =~ /^\s*#/ 
       # File format is: NAME ' '+ PATTERN '\n'
       name, pattern = line.gsub(/^\s*/, "").split(/\s+/, 2)
+      #p name => pattern
       # If the line is malformed, skip it.
       next if pattern.nil?
       # Trim newline and add the pattern.
@@ -70,9 +79,9 @@ class Grok
   def compile(pattern)
     @capture_map = {}
 
-    iterations_left = 1000
+    iterations_left = 10000
     @pattern = pattern
-    @expanded_pattern = pattern
+    @expanded_pattern = pattern.clone
     index = 0
 
     # Replace any instances of '%{FOO}' with that pattern.
@@ -151,8 +160,9 @@ class Grok
 
   private
   def init_discover
-    @discover = GrokDiscover.new(self)
-    @discover.logmask = logmask
+    require "grok/pure/discovery"
+    @discover = Grok::Discovery.new(self)
+    @discover.logger = @logger
   end # def init_discover
 
   public

data/lib/grok/pure/discovery.rb

@@ -0,0 +1,78 @@
+require "grok-pure"
+require "logger"
+
+class Grok::Discovery
+  attr_accessor :logger
+
+  def initialize(grok)
+    @grok = grok
+    @logger = Cabin::Channel.new
+    @logger.subscribe(Logger.new(STDOUT))
+    @logger.level = :warn
+  end # def initialize
+
+  def discover(text)
+    text = text.clone
+    # TODO(sissel): Sort patterns by complexity, most complex first.
+    #   - For each pattern, compile it in a grok by itself.
+    #   - Make a dictionary of { "name" => Grok } for each pattern
+    #   - Sort groks by complexity of the Grok#expanded_pattern
+    groks = {}
+    @grok.patterns.each do |name, expression| 
+      grok = Grok.new
+      # Copy in the same grok patterns from the parent
+      grok.patterns.merge!(@grok.patterns)
+      grok.compile("%{#{name}}")
+      groks[name] = grok
+    end
+
+    patterns = groks.sort { |a, b| compare(a, b) }
+
+    done = false
+    while !done
+      done = true # will reset this if we are not done later.
+      patterns.each do |name, grok|
+        # Skip patterns that lack complexity (SPACE, NOTSPACE, DATA, etc)
+        next if complexity(grok.expanded_pattern) < 20
+        m = grok.match(text)
+        # Skip non-matches
+        next unless m 
+        part = text[m.start ... m.end]
+        # Only include things that have word boundaries (not just words)
+        next if part !~ /.\b./
+        # Skip over parts that appear to include %{pattern} already
+        next if part =~ /%{[^}+]}/
+        acting = true
+        text[m.start ... m.end] = "%{#{name}}"
+
+        # Start the loop over again
+        done = false
+        break
+      end
+    end
+
+    return text
+  end # def discover
+
+  private
+  def compare(a, b)
+    # a and be are each: [ name, grok ]
+    # sort highest complexity first
+    return complexity(b.last.expanded_pattern) <=> complexity(a.last.expanded_pattern)
+  end # def compare
+
+  private
+  def complexity(expression)
+    score = expression.count("|") # number of branches in the pattern
+    score += expression.length # the length of the pattern
+  end # def complexity
+
+end # class Grok::Discovery
+
+#/* Compute the relative complexity of a pattern */
+#static int complexity(const grok_t *grok) {
+  #int score;
+  #score += string_count(grok->full_pattern, "|");
+  #score += strlen(grok->full_pattern) / 2;
+  #return -score; /* Sort most-complex first */
+#}

metadata

@@ -1,90 +1,66 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: jls-grok
-version: !ruby/object:Gem::Version 
-  hash: 55
+version: !ruby/object:Gem::Version
+  version: 0.10.0
   prerelease: 
-  segments: 
-  - 0
-  - 9
-  - 6
-  version: 0.9.6
 platform: ruby
-authors: 
+authors:
 - Jordan Sissel
 - Pete Fritchman
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-01-17 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-02-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: cabin
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &20862820 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - "="
-      - !ruby/object:Gem::Version 
-        hash: 21
-        segments: 
-        - 0
-        - 1
-        - 7
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
         version: 0.1.7
   type: :runtime
-  version_requirements: *id001
+  prerelease: false
+  version_requirements: *20862820
 description: Grok ruby bindings - pattern match/extraction tool
-email: 
+email:
 - jls@semicomplete.com
 - petef@databits.net
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - lib/grok-pure.rb
 - lib/grok.rb
 - lib/grok/c-ext/pile.rb
 - lib/grok/c-ext/match.rb
+- lib/grok/pure/discovery.rb
 - lib/grok/pure/pile.rb
 - lib/grok/pure/match.rb
-- lib/Grok.rb
 homepage: http://code.google.com/p/semicomplete/wiki/Grok
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.8.13
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: grok bindings for ruby
 test_files: []
-

data/lib/Grok.rb

@@ -1,3 +0,0 @@
-require "grok.rb"
-
-# compat for when grok was Grok.so