jls-grok 0.10.12 → 0.10.13

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2f35ecc4902d02d2ea98af056466222dd36b33dc
+  data.tar.gz: 916b52edd7029e5711ecc5a5489e2fa642f1cc8d
+SHA512:
+  metadata.gz: 404736a67e44293fd072047a6302bec733c72af889c2fb67a6c1cf8965a4f55c829c03ed5f0e6cd3e9545a77b9a21f6716f58aee8d7223ea9be4a90a8c620c04
+  data.tar.gz: 6680165222e03342034833cfe917c69d3fef8752894c716e277e0ae11b460f3f252c81445650612cb369d217e993cb5804597f6f5e0e862d390d3b48c1782c8b

data/lib/grok-pure.rb

@@ -1,7 +1,8 @@
 require "rubygems"
 require "logger"
-require "grok/pure/discovery"
 require "cabin"
+require "grok/pure/discovery"
+require "grok/pure/match"
 
 # TODO(sissel): Check if 'grok' c-ext has been loaded and abort?
 class Grok
@@ -9,7 +10,7 @@ class Grok
 
   # The pattern input
   attr_accessor :pattern
-  
+
   # The fully-expanded pattern (in regex form)
   attr_accessor :expanded_pattern
 
@@ -18,7 +19,7 @@ class Grok
 
   # The dictionary of pattern names to pattern expressions
   attr_accessor :patterns
-  
+
   PATTERN_RE = \
     /%\{    # match '%{' not prefixed with '\'
        (?<name>     # match the pattern name
@@ -50,6 +51,8 @@ class Grok
     @logger = Cabin::Channel.new
     @logger.subscribe(Logger.new(STDOUT))
     @logger.level = :warn
+    # Captures Lambda which is generated at Grok compile time and called at match time
+    @captures_func = nil
 
     # TODO(sissel): Throw exception if we aren't using Ruby 1.9.2 or newer.
   end # def initialize
@@ -66,7 +69,7 @@ class Grok
     file = File.new(path, "r")
     file.each do |line|
       # Skip comments
-      next if line =~ /^\s*#/ 
+      next if line =~ /^\s*#/
       # File format is: NAME ' '+ PATTERN '\n'
       name, pattern = line.gsub(/^\s*/, "").split(/\s+/, 2)
       #p name => pattern
@@ -76,16 +79,15 @@ class Grok
       add_pattern(name, pattern.chomp)
     end
     return nil
+  ensure
+    file.close
   end # def add_patterns_from_file
 
   public
-  def compile(pattern)
-    @capture_map = {}
-
+  def compile(pattern, named_captures_only=false)
     iterations_left = 10000
     @pattern = pattern
     @expanded_pattern = pattern.clone
-    index = 0
 
     # Replace any instances of '%{FOO}' with that pattern.
     loop do
@@ -101,61 +103,87 @@ class Grok
       end
 
       if @patterns.include?(m["pattern"])
-        # create a named capture index that we can push later as the named
-        # pattern. We do this because ruby regexp can't capture something
-        # by the same name twice.
         regex = @patterns[m["pattern"]]
-        #puts "patterns[#{m["pattern"]}] => #{regex}"
+        name = m["name"]
 
-        capture = "a#{index}" # named captures have to start with letters?
-        #capture = "%04d" % "#{index}" # named captures have to start with letters?
-        replacement_pattern = "(?<#{capture}>#{regex})"
-        @capture_map[capture] = m["name"]
-
-        #puts "Before: #{@expanded_pattern}"
-        #puts "m[0]: #{m[0]}"
-        #puts "replacement_pattern => #{replacement_pattern}"
-        #puts "Proposed: #{@expanded_pattern.sub(m[0], replacement_pattern)}"
+        if named_captures_only && name.index(":").nil?
+          # this has no semantic (pattern:foo) so we don't need to capture
+          replacement_pattern = "(?:#{regex})"
+        else
+          replacement_pattern = "(?<#{name}>#{regex})"
+        end
 
         # Ruby's String#sub() has a bug (or misfeature) that causes it to do bad
         # things to backslashes in string replacements, so let's work around it
         # See this gist for more details: https://gist.github.com/1491437
         # This hack should resolve LOGSTASH-226.
         @expanded_pattern.sub!(m[0]) { |s| replacement_pattern }
-
-        #puts "After: #{@expanded_pattern}"
-        #puts "m[0]: #{m[0]}"
-        #puts "replacement_pattern => #{replacement_pattern}"
-        index += 1
+        @logger.debug? and @logger.debug("replacement_pattern => #{replacement_pattern}")
       else
         raise PatternError, "pattern #{m[0]} not defined"
       end
     end
 
-    #@logger.debug("Finished expanding", :string => @expanded_pattern)
-    #puts "Expanded: #{@expanded_pattern}"
-    @regexp = Regexp.new(@expanded_pattern)
-    @logger.debug("Grok compiled OK", :pattern => pattern,
-                  :expanded_pattern => @expanded_pattern)
-  end # def compile
+    @regexp = Regexp.new(@expanded_pattern, Regexp::MULTILINE)
+    @logger.debug? and @logger.debug("Grok compiled OK", :pattern => pattern,
+                                     :expanded_pattern => @expanded_pattern)
+
+    @captures_func = compile_captures_func(@regexp)
+  end
+
+  private
+  # compiles the captures lambda so runtime match can be optimized
+  def compile_captures_func(re)
+    re_match = ["lambda do |match, &block|"]
+    re.named_captures.each do |name, indices|
+      pattern, name, coerce = name.split(":")
+      indices.each do |index|
+        coerce = case coerce
+                   when "int"; ".to_i"
+                   when "float"; ".to_f"
+                   else; ""
+                 end
+        name = pattern if name.nil?
+        re_match << "  block.call(#{name.inspect}, match[#{index}]#{coerce})"
+      end
+    end
+    re_match << "end"
+    return eval(re_match.join("\n"))
+  end # def compile_captures_func
 
   public
   def match(text)
     match = @regexp.match(text)
-
     if match
       grokmatch = Grok::Match.new
       grokmatch.subject = text
-      grokmatch.start, grokmatch.end = match.offset(0)
       grokmatch.grok = self
       grokmatch.match = match
-      @logger.debug("Regexp match object", :names => match.names, :captures => match.captures)
+      @logger.debug? and @logger.debug("Regexp match object", :names => match.names,
+                                       :captures => match.captures)
       return grokmatch
     else
       return false
     end
   end # def match
 
+  # Optimized match and capture instead of calling them separately
+  def match_and_capture(text)
+    match = @regexp.match(text)
+    if match
+      @logger.debug? and @logger.debug("Regexp match object", :names => match.names,
+                                       :captures => match.captures)
+      @captures_func.call(match) { |k,v| yield k,v }
+      return true
+    else
+      return false
+    end
+  end # def match_and_capture
+
+  def capture(match, block)
+    @captures_func.call(match) { |k,v| block.call k,v }
+  end # def capture
+
   public
   def discover(input)
     init_discover if @discover == nil
@@ -170,11 +198,4 @@ class Grok
     @discover.logger = @logger
   end # def init_discover
 
-  public
-  def capture_name(id)
-    return @capture_map[id]
-  end # def capture_name
 end # Grok
-
-require "grok/pure/match"
-require "grok/pure/pile"

data/lib/grok/pure/match.rb

@@ -2,8 +2,6 @@ require "grok-pure"
 
 class Grok::Match
   attr_accessor :subject
-  attr_accessor :start
-  attr_accessor :end
   attr_accessor :grok
   attr_accessor :match
 
@@ -13,16 +11,8 @@ class Grok::Match
   end
 
   public
-  def each_capture
-    @captures = Hash.new { |h, k| h[k] = Array.new }
-
-    #p :expanded => @grok.expanded_pattern
-    #p :map => @grok.capture_map
-    @match.names.zip(@match.captures).each do |id, value|
-      name = @grok.capture_name(id) || "_:#{id}"
-      yield name, value
-    end
-
+  def each_capture(&block)
+    @grok.capture(@match, block)
   end # def each_capture
 
   public

metadata

@@ -1,33 +1,30 @@
 --- !ruby/object:Gem::Specification
 name: jls-grok
 version: !ruby/object:Gem::Version
-  version: 0.10.12
-  prerelease: 
+  version: 0.10.13
 platform: ruby
 authors:
 - Jordan Sissel
 - Pete Fritchman
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-10-07 00:00:00.000000000 Z
+date: 2015-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cabin
-  requirement: !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.6.0
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.6.0
+  prerelease: false
+  type: :runtime
 description: Grok ruby bindings - pattern match/extraction tool
 email:
 - jls@semicomplete.com
@@ -36,37 +33,35 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/grok-pure.rb
 - lib/grok.rb
+- lib/grok/c-ext/match.rb
+- lib/grok/c-ext/pile.rb
 - lib/grok/namespace.rb
 - lib/grok/pure/discovery.rb
-- lib/grok/pure/pile.rb
 - lib/grok/pure/match.rb
-- lib/grok/c-ext/pile.rb
-- lib/grok/c-ext/match.rb
-- lib/grok-pure.rb
 homepage: http://code.google.com/p/semicomplete/wiki/Grok
 licenses: []
-post_install_message: 
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.25
-signing_key: 
-specification_version: 3
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
 summary: grok bindings for ruby
 test_files: []

data/lib/grok/pure/pile.rb

@@ -1,71 +0,0 @@
-require "grok-pure"
-require "logger"
-require "cabin"
-
-# A grok pile is an easy way to have multiple patterns together so
-# that you can try to match against each one.
-# The API provided should be similar to the normal Grok
-# interface, but you can compile multiple patterns and match will
-# try each one until a match is found.
-class Grok
-  class Pile
-    attr_accessor :logger
-
-    def initialize
-      @groks = []
-      @patterns = {}
-      @pattern_files = []
-      @logger = Cabin::Channel.new
-      @logger.subscribe(Logger.new(STDOUT))
-      @logger.level = :warn
-    end # def initialize
-
-    def logger=(logger)
-      @logger = logger
-      @groks.each { |g| g.logger = logger }
-    end # def logger=
-
-    # see Grok#add_pattern
-    def add_pattern(name, string)
-      @patterns[name] = string
-    end # def add_pattern
-
-    # see Grok#add_patterns_from_file
-    def add_patterns_from_file(path)
-      if !File.exists?(path)
-        raise "File does not exist: #{path}"
-      end
-      @pattern_files << path
-    end # def add_patterns_from_file
-
-    # see Grok#compile
-    def compile(pattern)
-      grok = Grok.new
-      grok.logger = @logger unless @logger.nil?
-      @patterns.each do |name, value|
-        grok.add_pattern(name, value)
-      end
-      @pattern_files.each do |path|
-        grok.add_patterns_from_file(path)
-      end
-      grok.compile(pattern)
-      @logger.info("Pile compiled new grok", :pattern => pattern,
-                   :expanded_pattern => grok.expanded_pattern)
-      @groks << grok
-    end # def compile
-
-    # Slight difference from Grok#match in that it returns
-    # the Grok instance that matched successfully in addition
-    # to the GrokMatch result.
-    # See also: Grok#match
-    def match(string)
-      @groks.each do |grok|
-        match = grok.match(string)
-        if match
-          return [grok, match]
-        end
-      end
-      return false
-    end # def match
-  end # class Pile
-end # class Grok