logstash-filter-grok 3.3.1 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99850448c56dc0d262fada37d7682318118e0541
-  data.tar.gz: b2de3eef13168f3831aa34b6899a7c5b40718358
+  metadata.gz: b1e652d5cf4cc9eff9c8a678a3c981b394c26de0
+  data.tar.gz: 6124306fb38c72fdad65262fb0e544ca5b51592b
 SHA512:
-  metadata.gz: 54b4579e88698e11f578a831a42ce1c352685abedb821b4da2cded069735068d373fe58e43ee1f6416ca70e58f69a1ed49df1e59f15340405ece5e25028cb14c
-  data.tar.gz: 00e36a17349e31d1f10c4081351c49c63f91b5f05f915ee64aa97a4ad27a0a819a8aef724c95c4773503ad0006271004b19e598f223fab82f9cfc6111f4bdfc6
+  metadata.gz: 2fd4bb01edc17528e22cf0085b8a09b5f42624c4f843c1cef1ff089c3ec3f8a4ba79a3e5957a5cec0d819a257e389c6c0985bfb1065b305ad7c8128435f89916
+  data.tar.gz: 0fbc997375ccbc3e5406b36afe73d4ee38f5e618c5f7e5c2158a5407d030e5c1a924f9e7fc4e220eed5dbd7276f3e01e84ddb1780ce8c195f19eebb91cd288ec

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.4.0
+  - Add option to define patterns inline in the filter using `pattern_definitions` configuration.
+
 ## 3.3.1
   - Docs: indicate that grok patterns are loaded when the pipeline is created
 

data/lib/logstash/filters/grok.rb

@@ -136,6 +136,11 @@
   #
   # The `timestamp`, `logsource`, `program`, and `pid` fields come from the
   # `SYSLOGBASE` pattern which itself is defined by other patterns.
+  #
+  # Another option is to define patterns _inline_ in the filter using `pattern_definitions`.
+  # This is mostly for convenience and allows user to define a pattern which can be used just in that 
+  # filter. This newly defined patterns in `pattern_definitions` will not be available outside of that particular `grok` filter.
+  #
   class LogStash::Filters::Grok < LogStash::Filters::Base
     config_name "grok"
     require "logstash/filters/grok/timeout_enforcer"
@@ -178,6 +183,12 @@
     # The patterns are loaded when the pipeline is created.
     config :patterns_dir, :validate => :array, :default => []
 
+    # A hash of pattern-name and pattern tuples defining custom patterns to be used by 
+    # the current filter. Patterns matching existing names will override the pre-existing 
+    # definition. Think of this as inline patterns available just for this definition of 
+    # grok
+    config :pattern_definitions, :validate => :hash, :default => {}
+
     # Glob pattern, used to select the pattern files in the directories
     # specified by patterns_dir
     config :patterns_files_glob, :validate => :string, :default => "*"
@@ -271,6 +282,7 @@
           grok = Grok.new
           grok.logger = @logger unless @logger.nil?
           add_patterns_from_files(@patternfiles, grok)
+          add_patterns_from_inline_definition(@pattern_definitions, grok)
           grok.compile(pattern, @named_captures_only)
           @patterns[field] << grok
         end
@@ -394,6 +406,14 @@
       end
     end # def add_patterns_from_files
 
+    private
+    def add_patterns_from_inline_definition(pattern_definitions, grok)
+      pattern_definitions.each do |name, pattern|
+        next if pattern.nil?
+        grok.add_pattern(name, pattern.chomp)
+      end
+    end
+
     def close
       @timeout_enforcer.stop!
     end

data/logstash-filter-grok.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-grok'
-  s.version         = '3.3.1'
+  s.version         = '3.4.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parse arbitrary text and structure it."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/grok_spec.rb

@@ -868,4 +868,40 @@ describe LogStash::Filters::Grok do
     end
   end
 
+  describe  "grok with inline pattern definition successfully extracts fields" do
+    config <<-CONFIG
+      filter {
+        grok {
+          match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level} %{MY_PATTERN:hindsight}" }
+          pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
+           "MY_PATTERN" => "%{YEAR}"}
+        }
+      }
+    CONFIG
+
+    sample "Mon Dec 26 16:22:08 2016 error 2020" do
+      insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
+      insist { subject.get("level") } == "error"
+      insist { subject.get("hindsight") } == "2020"
+    end
+  end
+
+  describe  "grok with inline pattern definition overwrites existing pattern definition" do
+    config <<-CONFIG
+      filter {
+        grok {
+          match => { "message" => "%{APACHE_TIME:timestamp} %{LOGLEVEL:level}" }
+          # loglevel was previously ([Aa]lert|ALERT|[Tt]...
+          pattern_definitions => { "APACHE_TIME" => "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
+           "LOGLEVEL" => "%{NUMBER}"}
+        }
+      }
+    CONFIG
+
+    sample "Mon Dec 26 16:22:08 2016 9999" do
+      insist { subject.get("timestamp") } == "Mon Dec 26 16:22:08 2016"
+      insist { subject.get("level") } == "9999"
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-grok
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-26 00:00:00.000000000 Z
+date: 2017-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement