logstash-filter-grok 3.4.0 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1e652d5cf4cc9eff9c8a678a3c981b394c26de0
-  data.tar.gz: 6124306fb38c72fdad65262fb0e544ca5b51592b
+  metadata.gz: 20ccae49d2cac575daa26bbda6f8554f1f3abd22
+  data.tar.gz: c90ffb04ace29dbe59a48f3b7db70cb60b72bc68
 SHA512:
-  metadata.gz: 2fd4bb01edc17528e22cf0085b8a09b5f42624c4f843c1cef1ff089c3ec3f8a4ba79a3e5957a5cec0d819a257e389c6c0985bfb1065b305ad7c8128435f89916
-  data.tar.gz: 0fbc997375ccbc3e5406b36afe73d4ee38f5e618c5f7e5c2158a5407d030e5c1a924f9e7fc4e220eed5dbd7276f3e01e84ddb1780ce8c195f19eebb91cd288ec
+  metadata.gz: 81f30c18b7f29b68f915f554482a7f44bf5f9e315c7cc3c3d0b70462f00c92162d34fd51acfa13459982b3787369ee580d3153385a8056dd226441bfc08e0ada
+  data.tar.gz: 58fa40252932500b01561e802d6889908d7485aabe5d932460fc841edc849d48b98968e1331d577887fe360c85d08ad3845c81581b170699f3fadd0a8b488901

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.4.1
+  - Fix subdirectories in a pattern folder causing an exception in some cases
+
 ## 3.4.0
   - Add option to define patterns inline in the filter using `pattern_definitions` configuration.
 

data/docs/index.asciidoc

@@ -0,0 +1,332 @@
+:plugin: grok
+:type: filter
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Grok
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+Parse arbitrary text and structure it.
+
+Grok is currently the best way in logstash to parse crappy unstructured log
+data into something structured and queryable.
+
+This tool is perfect for syslog logs, apache and other webserver logs, mysql
+logs, and in general, any log format that is generally written for humans
+and not computer consumption.
+
+Logstash ships with about 120 patterns by default. You can find them here:
+<https://github.com/logstash-plugins/logstash-patterns-core/tree/master/patterns>. You can add
+your own trivially. (See the `patterns_dir` setting)
+
+If you need help building patterns to match your logs, you will find the
+<http://grokdebug.herokuapp.com> and <http://grokconstructor.appspot.com/> applications quite useful!
+
+==== Grok Basics
+
+Grok works by combining text patterns into something that matches your
+logs.
+
+The syntax for a grok pattern is `%{SYNTAX:SEMANTIC}`
+
+The `SYNTAX` is the name of the pattern that will match your text. For
+example, `3.44` will be matched by the `NUMBER` pattern and `55.3.244.1` will
+be matched by the `IP` pattern. The syntax is how you match.
+
+The `SEMANTIC` is the identifier you give to the piece of text being matched.
+For example, `3.44` could be the duration of an event, so you could call it
+simply `duration`. Further, a string `55.3.244.1` might identify the `client`
+making a request.
+
+For the above example, your grok filter would look something like this:
+[source,ruby]
+%{NUMBER:duration} %{IP:client}
+
+Optionally you can add a data type conversion to your grok pattern. By default
+all semantics are saved as strings. If you wish to convert a semantic's data type,
+for example change a string to an integer then suffix it with the target data type.
+For example `%{NUMBER:num:int}` which converts the `num` semantic from a string to an
+integer. Currently the only supported conversions are `int` and `float`.
+
+.Examples:
+
+With that idea of a syntax and semantic, we can pull out useful fields from a
+sample log like this fictional http request log:
+[source,ruby]
+    55.3.244.1 GET /index.html 15824 0.043
+
+The pattern for this could be:
+[source,ruby]
+    %{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}
+
+A more realistic example, let's read these logs from a file:
+[source,ruby]
+    input {
+      file {
+        path => "/var/log/http.log"
+      }
+    }
+    filter {
+      grok {
+        match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
+      }
+    }
+
+After the grok filter, the event will have a few extra fields in it:
+
+* `client: 55.3.244.1`
+* `method: GET`
+* `request: /index.html`
+* `bytes: 15824`
+* `duration: 0.043`
+
+==== Regular Expressions
+
+Grok sits on top of regular expressions, so any regular expressions are valid
+in grok as well. The regular expression library is Oniguruma, and you can see
+the full supported regexp syntax https://github.com/kkos/oniguruma/blob/master/doc/RE[on the Oniguruma
+site].
+
+==== Custom Patterns
+
+Sometimes logstash doesn't have a pattern you need. For this, you have
+a few options.
+
+First, you can use the Oniguruma syntax for named capture which will
+let you match a piece of text and save it as a field:
+[source,ruby]
+    (?<field_name>the pattern here)
+
+For example, postfix logs have a `queue id` that is an 10 or 11-character
+hexadecimal value. I can capture that easily like this:
+[source,ruby]
+    (?<queue_id>[0-9A-F]{10,11})
+
+Alternately, you can create a custom patterns file.
+
+* Create a directory called `patterns` with a file in it called `extra`
+  (the file name doesn't matter, but name it meaningfully for yourself)
+* In that file, write the pattern you need as the pattern name, a space, then
+  the regexp for that pattern.
+
+For example, doing the postfix queue id example as above:
+[source,ruby]
+    # contents of ./patterns/postfix:
+    POSTFIX_QUEUEID [0-9A-F]{10,11}
+
+Then use the `patterns_dir` setting in this plugin to tell logstash where
+your custom patterns directory is. Here's a full example with a sample log:
+[source,ruby]
+    Jan  1 06:25:43 mailserver14 postfix/cleanup[21403]: BEF25A72965: message-id=<20130101142543.5828399CCAF@mailserver14.example.com>
+[source,ruby]
+    filter {
+      grok {
+        patterns_dir => ["./patterns"]
+        match => { "message" => "%{SYSLOGBASE} %{POSTFIX_QUEUEID:queue_id}: %{GREEDYDATA:syslog_message}" }
+      }
+    }
+
+The above will match and result in the following fields:
+
+* `timestamp: Jan  1 06:25:43`
+* `logsource: mailserver14`
+* `program: postfix/cleanup`
+* `pid: 21403`
+* `queue_id: BEF25A72965`
+* `syslog_message: message-id=<20130101142543.5828399CCAF@mailserver14.example.com>`
+
+The `timestamp`, `logsource`, `program`, and `pid` fields come from the
+`SYSLOGBASE` pattern which itself is defined by other patterns.
+
+Another option is to define patterns _inline_ in the filter using `pattern_definitions`.
+This is mostly for convenience and allows user to define a pattern which can be used just in that 
+filter. This newly defined patterns in `pattern_definitions` will not be available outside of that particular `grok` filter.
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Grok Filter Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-break_on_match>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-keep_empty_captures>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-match>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-named_captures_only>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-overwrite>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-pattern_definitions>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-patterns_dir>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-patterns_files_glob>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-tag_on_timeout>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-timeout_millis>> |<<number,number>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-common-options>> for a list of options supported by all
+filter plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-break_on_match"]
+===== `break_on_match` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Break on first match. The first successful match by grok will result in the
+filter being finished. If you want grok to try all patterns (maybe you are
+parsing different things), then set this to false.
+
+[id="plugins-{type}s-{plugin}-keep_empty_captures"]
+===== `keep_empty_captures` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+If `true`, keep empty captures as event fields.
+
+[id="plugins-{type}s-{plugin}-match"]
+===== `match` 
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+A hash of matches of field => value
+
+For example:
+[source,ruby]
+    filter {
+      grok { match => { "message" => "Duration: %{NUMBER:duration}" } }
+    }
+
+If you need to match multiple patterns against a single field, the value can be an array of patterns
+[source,ruby]
+    filter {
+      grok { match => { "message" => [ "Duration: %{NUMBER:duration}", "Speed: %{NUMBER:speed}" ] } }
+    }
+
+
+[id="plugins-{type}s-{plugin}-named_captures_only"]
+===== `named_captures_only` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+If `true`, only store named captures from grok.
+
+[id="plugins-{type}s-{plugin}-overwrite"]
+===== `overwrite` 
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+The fields to overwrite.
+
+This allows you to overwrite a value in a field that already exists.
+
+For example, if you have a syslog line in the `message` field, you can
+overwrite the `message` field with part of the match like so:
+[source,ruby]
+    filter {
+      grok {
+        match => { "message" => "%{SYSLOGBASE} %{DATA:message}" }
+        overwrite => [ "message" ]
+      }
+    }
+
+In this case, a line like `May 29 16:37:11 sadness logger: hello world`
+will be parsed and `hello world` will overwrite the original message.
+
+[id="plugins-{type}s-{plugin}-pattern_definitions"]
+===== `pattern_definitions` 
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+A hash of pattern-name and pattern tuples defining custom patterns to be used by 
+the current filter. Patterns matching existing names will override the pre-existing 
+definition. Think of this as inline patterns available just for this definition of 
+grok
+
+[id="plugins-{type}s-{plugin}-patterns_dir"]
+===== `patterns_dir` 
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+
+Logstash ships by default with a bunch of patterns, so you don't
+necessarily need to define this yourself unless you are adding additional
+patterns. You can point to multiple pattern directories using this setting.
+Note that Grok will read all files in the directory matching the patterns_files_glob
+and assume it's a pattern file (including any tilde backup files). 
+[source,ruby]
+    patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
+
+Pattern files are plain text with format:
+[source,ruby]
+    NAME PATTERN
+
+For example:
+[source,ruby]
+    NUMBER \d+
+
+The patterns are loaded when the pipeline is created.
+
+[id="plugins-{type}s-{plugin}-patterns_files_glob"]
+===== `patterns_files_glob` 
+
+  * Value type is <<string,string>>
+  * Default value is `"*"`
+
+Glob pattern, used to select the pattern files in the directories
+specified by patterns_dir
+
+[id="plugins-{type}s-{plugin}-tag_on_failure"]
+===== `tag_on_failure` 
+
+  * Value type is <<array,array>>
+  * Default value is `["_grokparsefailure"]`
+
+Append values to the `tags` field when there has been no
+successful match
+
+[id="plugins-{type}s-{plugin}-tag_on_timeout"]
+===== `tag_on_timeout` 
+
+  * Value type is <<string,string>>
+  * Default value is `"_groktimeout"`
+
+Tag to apply if a grok regexp times out.
+
+[id="plugins-{type}s-{plugin}-timeout_millis"]
+===== `timeout_millis` 
+
+  * Value type is <<number,number>>
+  * Default value is `30000`
+
+Attempt to terminate regexps after this amount of time.
+This applies per pattern if multiple patterns are applied
+This will never timeout early, but may take a little longer to timeout.
+Actual timeout is approximate based on a 250ms quantization.
+Set to 0 to disable timeouts
+
+
+
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/grok.rb

@@ -390,7 +390,11 @@
 
         Dir.glob(path).each do |file|
           @logger.trace("Grok loading patterns from file", :path => file)
-          patternfiles << file
+          if File.directory?(file)
+            @logger.debug("Skipping path because it is a directory", :path => file)
+          else
+            patternfiles << file
+          end
         end
       end
       patternfiles

data/lib/logstash/filters/grok/timeout_enforcer.rb

@@ -11,7 +11,7 @@ class LogStash::Filters::Grok::TimeoutEnforcer
     # Stores running matches with their start time, this is used to cancel long running matches
     # Is a map of Thread => start_time
     @threads_to_start_time = {}
-    @state_lock = java.util.concurrent.locks.ReentrantLock.new
+    @state_lock = ReentrantLock.new
   end
 
   def grok_till_timeout(event, grok, field, value)

data/logstash-filter-grok.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-grok'
-  s.version         = '3.4.0'
+  s.version         = '3.4.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parse arbitrary text and structure it."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
   s.add_runtime_dependency 'jls-grok', '~> 0.11.3'
+  s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_runtime_dependency 'logstash-patterns-core'
 
   s.add_development_dependency 'logstash-devutils'

data/spec/filters/grok_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
-
+require "stud/temporary"
 
 module LogStash::Environment
   # running the grok code outside a logstash package means
@@ -668,15 +668,13 @@ describe LogStash::Filters::Grok do
   end
 
   describe "patterns in the 'patterns/' dir override core patterns" do
-    require 'tmpdir'
-    require 'tempfile'
 
     let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
     let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
 
     before do
       FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file = Tempfile.new('grok', pattern_dir)
+      @file = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
       @file.write('WORD \b[2-5]\b')
       @file.close
     end
@@ -690,25 +688,23 @@ describe LogStash::Filters::Grok do
     end
 
     after do
-      @file.unlink
+      File.unlink @file
       FileUtils.rm_rf(pattern_dir) if has_pattern_dir?
     end
   end
 
   describe "patterns in custom dir override those in 'patterns/' dir" do
-    require 'tmpdir'
-    require 'tempfile'
 
-    let(:tmpdir) { Dir.mktmpdir }
+    let(:tmpdir) { Stud::Temporary.directory }
     let(:pattern_dir) { File.join(LogStash::Environment::LOGSTASH_HOME, "patterns") }
     let(:has_pattern_dir?) { Dir.exist?(pattern_dir) }
 
     before do
       FileUtils.mkdir(pattern_dir) unless has_pattern_dir?
-      @file1 = Tempfile.new('grok', pattern_dir)
+      @file1 = File.new(File.join(pattern_dir, 'grok.pattern'), 'w+')
       @file1.write('WORD \b[2-5]\b')
       @file1.close
-      @file2 = Tempfile.new('grok', tmpdir)
+      @file2 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
       @file2.write('WORD \b[0-1]\b')
       @file2.close
     end
@@ -722,24 +718,22 @@ describe LogStash::Filters::Grok do
     end
 
     after do
-      @file1.unlink
-      @file2.unlink
+      File.unlink @file1
+      File.unlink @file2
       FileUtils.remove_entry tmpdir
       FileUtils.rm_rf(pattern_dir) unless has_pattern_dir?
     end
   end
 
   describe "patterns with file glob" do
-    require 'tmpdir'
-    require 'tempfile'
 
-    let(:tmpdir) { Dir.mktmpdir(nil, "/tmp") }
+    let(:tmpdir) { Stud::Temporary.directory }
 
     before do
-      @file3 = Tempfile.new(['grok', '.pattern'], tmpdir)
+      @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
       @file3.write('WORD \b[0-1]\b')
       @file3.close
-      @file4 = Tempfile.new(['grok', '.pattern.old'], tmpdir)
+      @file4 = File.new(File.join(tmpdir, 'grok.pattern.old'), 'w+')
       @file4.write('WORD \b[2-5]\b')
       @file4.close
     end
@@ -753,8 +747,33 @@ describe LogStash::Filters::Grok do
     end
 
     after do
-      @file3.unlink
-      @file4.unlink
+      File.unlink @file3
+      File.unlink @file4
+      FileUtils.remove_entry tmpdir
+    end
+  end
+
+  describe "patterns with file glob on directory that contains subdirectories" do
+
+    let(:tmpdir) { Stud::Temporary.directory }
+
+    before do
+      @file3 = File.new(File.join(tmpdir, 'grok.pattern'), 'w+')
+      @file3.write('WORD \b[0-1]\b')
+      @file3.close
+      Dir.mkdir(File.join(tmpdir, "subdir"))
+    end
+
+    let(:config) do
+      "filter { grok { patterns_dir => \"#{tmpdir}\" patterns_files_glob => \"*\" match => { \"message\" => \"%{WORD:word}\" } } }"
+    end
+
+    sample("message" => '0') do
+      insist { subject.get("tags") } == nil
+    end
+
+    after do
+      File.unlink @file3
       FileUtils.remove_entry tmpdir
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-grok
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.4.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-01 00:00:00.000000000 Z
+date: 2017-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -44,6 +44,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.11.3
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.22
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -84,6 +98,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/filters/grok.rb
 - lib/logstash/filters/grok/timeout_enforcer.rb
 - lib/logstash/filters/grok/timeout_exception.rb