logstash-filter-grok 2.0.3 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c29b365e4050c830edadd3f1787b202851ab318
-  data.tar.gz: ab4929767f5244347793b383bb33a4ae8f86741d
+  metadata.gz: 4f63a7bd21e2ebab75a4cc94f26731323f8912f7
+  data.tar.gz: a2ae27830a99ac4b0bdeb2a2ff76c8e1f691e1f3
 SHA512:
-  metadata.gz: 41c8bb7ec629ab346b44013059e05ddc13bffaa2ca9eb25f1c8d1c8d730fb6272d6ae8d22e2651a99bd3d524d9fe870a51132558a0cda01c3a0cf3af05e4ae4b
-  data.tar.gz: 2825968293377f7ec3f0e6a327e6863e0bbd8c78f24100ab46f630f14f5e361006c2ff8e262efb3293544a769617dcc4644ad448a3eb972d40fdbd917973369d
+  metadata.gz: 872e7daf89f4af9d5d92ceea3ba791b24fd358e1a1f7e5d738032b2f90286b256ad544ae06a24231c290535526efaef3a772fbb81e09f991928b17521ce83563
+  data.tar.gz: 4a8ff939e59d6c704946618e315b17df72ae5fedaa2cdd9449652c22f4987f14458f31e15d8bbd9b63e6f0d371c85585143de1676b8e30485e528871a006a009

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.0.5
+  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+# 2.0.4
+  - New dependency requirements for logstash-core for the 5.0 release
 ## 2.0.3
  - fix fieldref assignment to avoid assumption on mutable object
 ## 2.0.0

data/lib/logstash/filters/grok.rb

@@ -83,7 +83,7 @@
   #
   # Grok sits on top of regular expressions, so any regular expressions are valid
   # in grok as well. The regular expression library is Oniguruma, and you can see
-  # the full supported regexp syntax https://github.com/kkos/oniguruma/blob/master/doc/RE[on the Onigiruma
+  # the full supported regexp syntax https://github.com/kkos/oniguruma/blob/master/doc/RE[on the Oniguruma
   # site].
   #
   # ==== Custom Patterns
@@ -120,7 +120,7 @@
   # [source,ruby]
   #     filter {
   #       grok {
-  #         patterns_dir => "./patterns"
+  #         patterns_dir => ["./patterns"]
   #         match => { "message" => "%{SYSLOGBASE} %{POSTFIX_QUEUEID:queue_id}: %{GREEDYDATA:syslog_message}" }
   #       }
   #     }
@@ -166,8 +166,8 @@
     # Logstash ships by default with a bunch of patterns, so you don't
     # necessarily need to define this yourself unless you are adding additional
     # patterns. You can point to multiple pattern directories using this setting
-    # Note that Grok will read all files in the directory and assume its a pattern
-    # file (including any tilde backup files)
+    # Note that Grok will read all files in the directory matching the patterns_files_glob
+    # and assume its a pattern file (including any tilde backup files)
     # [source,ruby]
     #     patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
     #
@@ -180,6 +180,10 @@
     #     NUMBER \d+
     config :patterns_dir, :validate => :array, :default => []
 
+    # Glob pattern, used to select the pattern files in the directories
+    # specified by patterns_dir
+    config :patterns_files_glob, :validate => :string, :default => "*"
+
     # Break on first match. The first successful match by grok will result in the
     # filter being finished. If you want grok to try all patterns (maybe you are
     # parsing different things), then set this to false.
@@ -241,19 +245,9 @@
 
       # Have @@patterns_path show first. Last-in pattern definitions win; this
       # will let folks redefine built-in patterns at runtime.
-      @patterns_dir = @@patterns_path.to_a + @patterns_dir
-      @logger.info? and @logger.info("Grok patterns path", :patterns_dir => @patterns_dir)
-      @patterns_dir.each do |path|
-        if File.directory?(path)
-          path = File.join(path, "*")
-        end
-
-        Dir.glob(path).each do |file|
-          @logger.info? and @logger.info("Grok loading patterns from file", :path => file)
-          @patternfiles << file
-        end
-      end
-
+      @patternfiles += patterns_files_from_paths(@@patterns_path.to_a, "*")
+      @patternfiles += patterns_files_from_paths(@patterns_dir, @patterns_files_glob)
+  
       @patterns = Hash.new { |h,k| h[k] = [] }
 
       @logger.info? and @logger.info("Match data", :match => @match)
@@ -352,6 +346,23 @@
       end
     end
 
+    private
+    def patterns_files_from_paths(paths, glob)
+      patternfiles = []
+      @logger.info? and @logger.info("Grok patterns path", :paths => paths)
+      paths.each do |path|
+        if File.directory?(path)
+          path = File.join(path, glob)
+        end
+
+        Dir.glob(path).each do |file|
+          @logger.info? and @logger.info("Grok loading patterns from file", :path => file)
+          patternfiles << file
+        end
+      end
+      patternfiles
+    end # def patterns_files_from_paths
+
     private
     def add_patterns_from_files(paths, grok)
       paths.each do |path|

data/logstash-filter-grok.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-grok'
-  s.version         = '2.0.3'
+  s.version         = '2.0.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parse arbitrary text and structure it."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
 
   s.add_runtime_dependency 'jls-grok', '~> 0.11.1'
   s.add_runtime_dependency 'logstash-patterns-core'

data/spec/filters/grok_spec.rb

@@ -1,11 +1,21 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 
-# running the grok code outside a logstash package means
-# LOGSTASH_HOME will not be defined, so let's set it here
-# before requiring the grok filter
-unless LogStash::Environment.const_defined?(:LOGSTASH_HOME)
-  LogStash::Environment::LOGSTASH_HOME = File.expand_path("../../../", __FILE__)
+
+module LogStash::Environment
+  # running the grok code outside a logstash package means
+  # LOGSTASH_HOME will not be defined, so let's set it here
+  # before requiring the grok filter
+  unless self.const_defined?(:LOGSTASH_HOME)
+    LOGSTASH_HOME = File.expand_path("../../../", __FILE__)
+  end
+
+  # also :pattern_path method must exist so we define it too
+  unless self.method_defined?(:pattern_path)
+    def pattern_path(path)
+      ::File.join(LOGSTASH_HOME, "patterns", path)
+    end
+  end
 end
 
 require "logstash/filters/grok"
@@ -661,7 +671,7 @@ describe LogStash::Filters::Grok do
 
     before do
       FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file = Tempfile.new('grok', pattern_dir);
+      @file = Tempfile.new('grok', pattern_dir)
       @file.write('WORD \b[2-5]\b')
       @file.close
     end
@@ -690,8 +700,12 @@ describe LogStash::Filters::Grok do
 
     before do
       FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file1 = Tempfile.new('grok', pattern_dir);  @file1.write('WORD \b[2-5]\b'); @file1.close
-      @file2 = Tempfile.new('grok', tmpdir);       @file2.write('WORD \b[0-1]\b'); @file2.close
+      @file1 = Tempfile.new('grok', pattern_dir)
+      @file1.write('WORD \b[2-5]\b')
+      @file1.close
+      @file2 = Tempfile.new('grok', tmpdir)
+      @file2.write('WORD \b[0-1]\b')
+      @file2.close
     end
 
     let(:config) do
@@ -710,6 +724,36 @@ describe LogStash::Filters::Grok do
     end
   end
 
+  describe "patterns with file glob" do
+    require 'tmpdir'
+    require 'tempfile'
+
+    let(:tmpdir) { Dir.mktmpdir(nil, "/tmp") }
+
+    before do
+      @file3 = Tempfile.new(['grok', '.pattern'], tmpdir)
+      @file3.write('WORD \b[0-1]\b')
+      @file3.close
+      @file4 = Tempfile.new(['grok', '.pattern.old'], tmpdir)
+      @file4.write('WORD \b[2-5]\b')
+      @file4.close
+    end
+
+    let(:config) do
+      "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*.pattern\" pattern => \"%{WORD:word}\" } }"
+    end
+
+    sample("message" => '0') do
+      insist { subject["tags"] } == nil
+    end
+
+    after do
+      @file3.unlink
+      @file4.unlink
+      FileUtils.remove_entry tmpdir
+    end
+  end
+
   describe  "grok with nil coerced value" do
     config <<-CONFIG
       filter {

metadata

@@ -1,77 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-grok
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.5
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-16 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '1.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: jls-grok
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.1
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.11.1
+  name: jls-grok
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-patterns-core
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.11.1
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-patterns-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -99,12 +93,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []