logstash-filter-kv 4.4.1 → 4.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f997d5d9787b39b3ffbb7f27130fbef6137ebf0f03ec65202795c47be856032
-  data.tar.gz: 20864a2cb4f477591f010e1e7bdc682db38615dedd76fb8d1e6d961c784dec9a
+  metadata.gz: da90e7e03c6cb4960a1c224d216f03ee985b6f1b320832b265b4abd112ac5ade
+  data.tar.gz: a4f50d4707f2a3c74cd917e8f86b1acc53e974b56e967953009c6b1f6d241ec3
 SHA512:
-  metadata.gz: e866ec418609a22c5365d737f0b3714a650b1d2a8538b9d59b709f4661ccf4af6e6a3a0a58eead83f6fa2dfa1596885c2ee642b6d1ac2a52005512d7a568c197
-  data.tar.gz: d101397b3fa8a9f488e56c87532ccf71d8c683344a7cf27d5befa7dcfcc5c3ea6299816648ef47c22ee248b9fecc989136c24193e122b577a165cfa0fe3fa662
+  metadata.gz: 6b2f78f389c0ae6d43132aff4fce9ff868dd044a220cd9aeaf69c5015d05c0a200be2c6fdc0df9876c4395912e8ff32ee2649e464a2d2d7dc70274243994f08e
+  data.tar.gz: 855fb4feebcb15be71505bef3ca4920b60d2e06c1adfd3ae646dc2caf5c95ac69609a6a7cc52ab327bcfce29f92cace5e47612b6a8af542016f468657a007b17

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 4.7.0
+ - Allow attaching multiple tags on failure. The `tag_on_failure` option now also supports an array of strings [#92](https://github.com/logstash-plugins/logstash-filter-kv/issues/92)
+
+## 4.6.0
+ - Added `allow_empty_values` option [#72](https://github.com/logstash-plugins/logstash-filter-kv/pull/72)
+
+## 4.5.0
+ - Feat: check that target is set in ECS mode [#96](https://github.com/logstash-plugins/logstash-filter-kv/pull/96)
+
 ## 4.4.1
  - Fixed issue where a `field_split_pattern` containing a literal backslash failed to match correctly [#87](https://github.com/logstash-plugins/logstash-filter-kv/issues/87)
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-filter-kv.svg)](https://travis-ci.org/logstash-plugins/logstash-filter-kv)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-filter-kv.svg)](https://travis-ci.com/logstash-plugins/logstash-filter-kv)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -44,6 +44,13 @@ in case your data is not structured using `=` signs and whitespace.
 For example, this filter can also be used to parse query parameters like
 `foo=bar&baz=fizz` by setting the `field_split` parameter to `&`.
 
+[id="plugins-{type}s-{plugin}-ecs_metadata"]
+==== Event Metadata and the Elastic Common Schema (ECS)
+
+The plugin behaves the same regardless of ECS compatibility, except giving a warning when ECS is enabled and `target` isn't set.
+
+TIP: Set the `target` option to avoid potential schema conflicts.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Kv Filter Configuration Options
 
@@ -53,7 +60,9 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-allow_duplicate_values>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-allow_empty_values>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-default_keys>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-exclude_keys>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-field_split>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-field_split_pattern>> |<<string,string>>|No
@@ -65,7 +74,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-remove_char_value>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-source>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-tag_on_failure>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-tag_on_timeout>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-timeout_millis>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-transform_key>> |<<string,string>>, one of `["lowercase", "uppercase", "capitalize"]`|No
@@ -101,6 +110,17 @@ you could use this configuration:
       }
     }
 
+[id="plugins-{type}s-{plugin}-allow_empty_values"]
+===== `allow_empty_values`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+A bool option for explicitly including empty values.
+When set to true, empty values will be added to the event.
+
+NOTE: Parsing empty values typically requires <<plugins-{type}s-{plugin}-whitespace,`whitespace => strict`>>.
+
 [id="plugins-{type}s-{plugin}-default_keys"]
 ===== `default_keys` 
 
@@ -117,6 +137,17 @@ in case these keys do not exist in the source field being parsed.
       }
     }
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: does not use ECS-compatible field names
+    ** `v1`: Elastic Common Schema compliant behavior (warns when `target` isn't set)
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
+
 [id="plugins-{type}s-{plugin}-exclude_keys"]
 ===== `exclude_keys` 
 
@@ -341,12 +372,12 @@ For example, to place all keys into the event field kv:
 [id="plugins-{type}s-{plugin}-tag_on_failure"]
 ===== `tag_on_failure`
 
-  * Value type is <<string,string>>
-  * The default value for this setting is `_kv_filter_error`.
+  * Value type is <<array,array>>
+  * The default value for this setting is [`_kv_filter_error`].
 
 When a kv operation causes a runtime exception to be thrown within the plugin,
 the operation is safely aborted without crashing the plugin, and the event is
-tagged with the provided value.
+tagged with the provided values.
 
 [id="plugins-{type}s-{plugin}-tag_on_timeout"]
 ===== `tag_on_timeout`

data/lib/logstash/filters/kv.rb

@@ -2,6 +2,9 @@
 
 require "logstash/filters/base"
 require "logstash/namespace"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 require "timeout"
 
 # This filter helps automatically parse messages (or specific event fields)
@@ -30,6 +33,11 @@ require "timeout"
 class LogStash::Filters::KV < LogStash::Filters::Base
   config_name "kv"
 
+  include LogStash::PluginMixins::ECSCompatibilitySupport
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   # Constants used for transform check
   TRANSFORM_LOWERCASE_KEY = "lowercase"
   TRANSFORM_UPPERCASE_KEY = "uppercase"
@@ -59,7 +67,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # These characters form a regex character class and thus you must escape special regex
   # characters like `[` or `]` using `\`.
   #
-  # Only leading and trailing characters are trimed from the key.
+  # Only leading and trailing characters are trimmed from the key.
   #
   # For example, to trim `<` `>` `[` `]` and `,` characters from keys:
   # [source,ruby]
@@ -201,7 +209,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # For example, to process the `not_the_message` field:
   # [source,ruby]
   #     filter { kv { source => "not_the_message" } }
-  config :source, :validate => :string, :default => "message"
+  config :source, :validate => :field_reference, :default => "message"
 
   # The name of the container to put all of the key-value pairs into.
   #
@@ -211,7 +219,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # For example, to place all keys into the event field kv:
   # [source,ruby]
   #     filter { kv { target => "kv" } }
-  config :target, :validate => :string
+  config :target, :validate => :field_reference
 
   # An array specifying the parsed keys which should be added to the event.
   # By default all keys will be added.
@@ -264,6 +272,9 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     }
   config :allow_duplicate_values, :validate => :boolean, :default => true
 
+  # A bool option for keeping empty or nil values.
+  config :allow_empty_values, :validate => :boolean, :default => false
+
   # A boolean specifying whether to treat square brackets, angle brackets,
   # and parentheses as value "wrappers" that should be removed from the value.
   # [source,ruby]
@@ -327,7 +338,10 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   config :tag_on_timeout, :validate => :string, :default => '_kv_filter_timeout'
 
   # Tag to apply if kv errors
-  config :tag_on_failure, :validate => :string, :default => '_kv_filter_error'
+  config :tag_on_failure, :validate => :array, :default => ['_kv_filter_error']
+
+
+  EMPTY_STRING = ''.freeze
 
   def register
     # Too late to set the regexp interruptible flag, at least warn if it is not set.
@@ -411,8 +425,8 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
     @logger.debug? && @logger.debug("KV scan regex", :regex => @scan_re.inspect)
 
-    # divide by float to allow fractionnal seconds, the Timeout class timeout value is in seconds but the underlying
-    # executor resolution is in microseconds so fractionnal second parameter down to microseconds is possible.
+    # divide by float to allow fractional seconds, the Timeout class timeout value is in seconds but the underlying
+    # executor resolution is in microseconds so fractional second parameter down to microseconds is possible.
     # see https://github.com/jruby/jruby/blob/9.2.7.0/core/src/main/java/org/jruby/ext/timeout/Timeout.java#L125
     @timeout_seconds = @timeout_millis / 1000.0
   end
@@ -429,7 +443,9 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     return if kv.empty?
 
     if @target
-      @logger.debug? && @logger.debug("Overwriting existing target field", :target => @target)
+      if event.include?(@target)
+        @logger.debug? && @logger.debug("Overwriting existing target field", field: @target, existing_value: event.get(@target))
+      end
       event.set(@target, kv)
     else
       kv.each{|k, v| event.set(k, v)}
@@ -444,7 +460,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     meta = { :exception => ex.message }
     meta[:backtrace] = ex.backtrace if logger.debug?
     logger.warn('Exception while parsing KV', meta)
-    event.tag(@tag_on_failure)
+    @tag_on_failure.each { |tag| event.tag(tag) }
   end
 
   def close
@@ -484,9 +500,9 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
     value = value.to_s
 
-    value.bytesize < 255 ? "`#{value}`" : "entry too large; first 255 chars are `#{value[0..255].dump}`"
+    value.bytesize < 255 ? "`#{value.dump}`" : "(entry too large to show; showing first 255 characters) `#{value[0..255].dump}`[...]"
   end
-  
+
   def has_value_splitter?(s)
     s =~ @value_split_re
   end
@@ -559,12 +575,12 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     exclude_keys = @exclude_keys.map{|key| event.sprintf(key)}
 
     text.scan(@scan_re) do |key, *value_candidates|
-      value = value_candidates.compact.first
-      next if value.nil? || value.empty?
+      value = value_candidates.compact.first || EMPTY_STRING
+      next if value.empty? && !@allow_empty_values
 
-      key = @trim_key ? key.gsub(@trim_key_re, "") : key
-      key = @remove_char_key ? key.gsub(@remove_char_key_re, "") : key
-      key = @transform_key ? transform(key, @transform_key) : key
+      key = key.gsub(@trim_key_re, EMPTY_STRING) if @trim_key
+      key = key.gsub(@remove_char_key_re, EMPTY_STRING) if @remove_char_key
+      key = transform(key, @transform_key) if @transform_key
 
       # Bail out as per the values of include_keys and exclude_keys
       next if not include_keys.empty? and not include_keys.include?(key)
@@ -573,9 +589,9 @@ class LogStash::Filters::KV < LogStash::Filters::Base
 
       key = event.sprintf(@prefix) + key
 
-      value = @trim_value ? value.gsub(@trim_value_re, "") : value
-      value = @remove_char_value ? value.gsub(@remove_char_value_re, "") : value
-      value = @transform_value ? transform(value, @transform_value) : value
+      value = value.gsub(@trim_value_re, EMPTY_STRING) if @trim_value
+      value = value.gsub(@remove_char_value_re, EMPTY_STRING) if @remove_char_value
+      value = transform(value, @transform_value) if @transform_value
 
       # Bail out if inserting duplicate value in key mapping when unique_values
       # option is set to true.

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '4.4.1'
+  s.version         = '4.7.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses key-value pairs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,6 +21,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'insist'

data/spec/filters/kv_spec.rb

@@ -460,7 +460,6 @@ describe LogStash::Filters::KV do
     end
   end
 
-
   describe "test data from specific sub source" do
     config <<-CONFIG
       filter {
@@ -519,7 +518,6 @@ describe LogStash::Filters::KV do
     end
   end
 
-
   describe "test data from specific sub source and target" do
     config <<-CONFIG
       filter {
@@ -721,6 +719,27 @@ describe LogStash::Filters::KV do
     end
   end
 
+  describe "Allowing empty values" do
+    config <<-CONFIG
+      filter {
+        kv {
+          field_split => " "
+          source => "source"
+          allow_empty_values => true
+          whitespace => strict
+        }
+      }
+    CONFIG
+
+    sample("source" => "present=one empty= emptyquoted='' present=two emptybracketed=[] endofinput=") do
+      insist { subject.get('[present]') } == ['one','two']
+      insist { subject.get('[empty]') } == ''
+      insist { subject.get('[emptyquoted]') } == ''
+      insist { subject.get('[emptybracketed]') } == ''
+      insist { subject.get('[endofinput]') } == ''
+    end
+  end
+
   describe "Allow duplicate key/value pairs by default" do
     config <<-CONFIG
       filter {
@@ -1038,7 +1057,6 @@ describe "multi character splitting" do
     it_behaves_like "parsing all fields and values"
   end
 
-
   context "example from @guyboertje in #15" do
     let(:message) { 'key1: val1; key2: val2; key3:  https://site/?g={......"...;  CLR  rv:11.0)"..}; key4: val4;' }
     let(:options) {
@@ -1131,6 +1149,17 @@ context 'runtime errors' do
         plugin.filter(event)
         expect(event.get('tags')).to_not be_nil
         expect(event.get('tags')).to include('KV-ERROR')
+        expect(event.get('tags')).to_not include('_kv_filter_error')
+      end
+    end
+    context 'when multiple custom tags are defined' do
+      let(:options) { super().merge("tag_on_failure" => ["kv_FAIL_one", "_kv_fail_TWO"])}
+      it 'tags the event with the custom tag' do
+        plugin.filter(event)
+        expect(event.get('tags')).to_not be_nil
+        expect(event.get('tags')).to include('kv_FAIL_one')
+        expect(event.get('tags')).to include('_kv_fail_TWO')
+        expect(event.get('tags')).to_not include('_kv_filter_error')
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 4.4.1
+  version: 4.7.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-03 00:00:00.000000000 Z
+date: 2022-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,34 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -97,8 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Parses key-value pairs