logstash-filter-kv 4.0.3 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3020b8c8f8399dfaaf9533b83ed66af6f90ab9843476d45e934feba3f48acea9
-  data.tar.gz: f5bde51c1f94ef0c376d4c1f9754eaca9da89daca9171326e16e2cd11db25fbe
+  metadata.gz: db1a1524254aafe6af64400a781c90fe02a0e82bcc037d8ce43b652de30dea93
+  data.tar.gz: 36443b889dfde58914512466fe3eadb703fca7c30b665a3d02106f5c6b91cada
 SHA512:
-  metadata.gz: 40b0548bdf2a9337529f9078fb330582a98860d35672fcd826f6e2d7356b8f0add71186e7ae98022441260f9788bf727318263dfdd649761ea9ec1a4e2e9b2eb
-  data.tar.gz: 35f1bc9f1af780f015946a7101392b7dc0a8c72c5a28e45edff46aa2cc40dab1c3b7d590b7c71c45d354a5c0814034e116411392432d73f7b3fbfa6f7e12aed9
+  metadata.gz: 072403211a806a928e240df180ab2180257b22579db3dd9de9d3ec4065f0e9b75374db3ade99c7e09ad0fc5b66f09701026ee97f81ec3a76c98826b50bd33e94
+  data.tar.gz: 8c7dd7066d669c4f0fea15942f14fc2c80a9412190e31bb0b36cb7382843b3f8839b843b89c72ba930eb04e697e7e5536b01603c78c0666db2b81a49948a0d3d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.0
+  - feature: add option to split fields and values using a regex pattern (#55)
+
 ## 4.0.3
   - Update gemspec summary
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -135,7 +135,7 @@ To exclude `from` and `to`, but retain the `foo` key, you could use this configu
   * Value type is <<string,string>>
   * Default value is `" "`
 
-A string of characters to use as delimiters for parsing out key-value pairs.
+A string of characters to use as single-character field delimiters for parsing out key-value pairs.
 
 These characters form a regex character class and thus you must escape special regex
 characters like `[` or `]` using `\`.
@@ -160,6 +160,29 @@ fields:
 * `oq: bobo`
 * `ss: 12345`
 
+[id="plugins-{type}s-{plugin}-field_split_pattern"]
+===== `field_split_pattern`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A regex expression to use as field delimiter for parsing out key-value pairs.
+Useful to define multi-character field delimiters.
+Setting the `field_split_pattern` options will take precedence over the `field_split` option.
+
+Note that you should avoid using captured groups in your regex and you should be
+cautious with lookaheads or lookbehinds and positional anchors.
+
+For example, to split fields on a repetition of one or more colons
+`k1=v1:k2=v2::k3=v3:::k4=v4`:
+[source,ruby]
+    filter { kv { field_split_pattern => ":+" } }
+
+To split fields on a regex character that need escaping like the plus sign
+`k1=v1++k2=v2++k3=v3++k4=v4`:
+[source,ruby]
+    filter { kv { field_split_pattern => "\\+\\+" } }
+
 [id="plugins-{type}s-{plugin}-include_brackets"]
 ===== `include_brackets` 
 
@@ -393,7 +416,7 @@ For example, to trim `<`, `>`, `[`, `]` and `,` characters from values:
   * Value type is <<string,string>>
   * Default value is `"="`
 
-A non-empty string of characters to use as delimiters for identifying key-value relations.
+A non-empty string of characters to use as single-character value delimiters for parsing out key-value pairs.
 
 These characters form a regex character class and thus you must escape special regex
 characters like `[` or `]` using `\`.
@@ -404,6 +427,21 @@ For example, to identify key-values such as
     filter { kv { value_split => ":" } }
 
 
+[id="plugins-{type}s-{plugin}-value_split_pattern"]
+===== `value_split_pattern`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A regex expression to use as value delimiter for parsing out key-value pairs.
+Useful to define multi-character value delimiters.
+Setting the `value_split_pattern` options will take precedence over the `value_split option`.
+
+Note that you should avoid using captured groups in your regex and you should be
+cautious with lookaheads or lookbehinds and positional anchors.
+
+See `field_split_pattern` for examples.
+
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/kv.rb

@@ -123,7 +123,7 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     }
   config :transform_key, :validate => [TRANSFORM_LOWERCASE_KEY, TRANSFORM_UPPERCASE_KEY, TRANSFORM_CAPITALIZE_KEY]
 
-  # A string of characters to use as delimiters for parsing out key-value pairs.
+  # A string of characters to use as single-character field delimiters for parsing out key-value pairs.
   #
   # These characters form a regex character class and thus you must escape special regex
   # characters like `[` or `]` using `\`.
@@ -149,8 +149,25 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   # * `ss: 12345`
   config :field_split, :validate => :string, :default => ' '
 
+  # A regex expression to use as field delimiter for parsing out key-value pairs.
+  # Useful to define multi-character field delimiters.
+  # Setting the field_split_pattern options will take precedence over the field_split option.
+  #
+  # Note that you should avoid using captured groups in your regex and you should be
+  # cautious with lookaheads or lookbehinds and positional anchors.
+  #
+  # For example, to split fields on a repetition of one or more colons
+  # `k1=v1:k2=v2::k3=v3:::k4=v4`:
+  # [source,ruby]
+  #     filter { kv { field_split_pattern => ":+" } }
+  #
+  # To split fields on a regex character that need escaping like the plus sign
+  # `k1=v1++k2=v2++k3=v3++k4=v4`:
+  # [source,ruby]
+  #     filter { kv { field_split_pattern => "\\+\\+" } }
+  config :field_split_pattern, :validate => :string
 
-  # A non-empty string of characters to use as delimiters for identifying key-value relations.
+  # A non-empty string of characters to use as single-character value delimiters for parsing out key-value pairs.
   #
   # These characters form a regex character class and thus you must escape special regex
   # characters like `[` or `]` using `\`.
@@ -161,6 +178,16 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   #     filter { kv { value_split => ":" } }
   config :value_split, :validate => :string, :default => '='
 
+  # A regex expression to use as value delimiter for parsing out key-value pairs.
+  # Useful to define multi-character value delimiters.
+  # Setting the value_split_pattern options will take precedence over the value_split option.
+  #
+  # Note that you should avoid using captured groups in your regex and you should be
+  # cautious with lookaheads or lookbehinds and positional anchors.
+  #
+  # See field_split_pattern for examples.
+  config :value_split_pattern, :validate => :string
+
   # A string to prepend to all of the extracted keys.
   #
   # For example, to prepend arg_ to all keys:
@@ -279,24 +306,70 @@ class LogStash::Filters::KV < LogStash::Filters::Base
   def register
     if @value_split.empty?
       raise LogStash::ConfigurationError, I18n.t(
-        "logstash.agent.configuration.invalid_plugin_register",
+        "logstash.runner.configuration.invalid_plugin_register",
         :plugin => "filter",
         :type => "kv",
         :error => "Configuration option 'value_split' must be a non-empty string"
       )
     end
 
+    if @field_split_pattern && @field_split_pattern.empty?
+      raise LogStash::ConfigurationError, I18n.t(
+          "logstash.runner.configuration.invalid_plugin_register",
+          :plugin => "filter",
+          :type => "kv",
+          :error => "Configuration option 'field_split_pattern' must be a non-empty string"
+      )
+    end
+
+    if @value_split_pattern && @value_split_pattern.empty?
+      raise LogStash::ConfigurationError, I18n.t(
+          "logstash.runner.configuration.invalid_plugin_register",
+          :plugin => "filter",
+          :type => "kv",
+          :error => "Configuration option 'value_split_pattern' must be a non-empty string"
+      )
+    end
+
     @trim_value_re = Regexp.new("^[#{@trim_value}]|[#{@trim_value}]$") if @trim_value
     @trim_key_re = Regexp.new("^[#{@trim_key}]|[#{@trim_key}]$") if @trim_key
 
     @remove_char_value_re = Regexp.new("[#{@remove_char_value}]") if @remove_char_value
     @remove_char_key_re = Regexp.new("[#{@remove_char_key}]") if @remove_char_key
 
-    valueRxString = "(?:\"([^\"]+)\"|'([^']+)'"
-    valueRxString += "|\\(([^\\)]+)\\)|\\[([^\\]]+)\\]|<([^>]+)>" if @include_brackets
-    valueRxString += "|((?:\\\\ |[^" + @field_split + "])+))"
-    @scan_re = Regexp.new("((?:\\\\ |[^" + @field_split + @value_split + "])+)\s*[" + @value_split + "]\s*" + valueRxString)
-    @value_split_re = /[#{@value_split}]/
+    field_split = Regexp::compile(@field_split_pattern || /[#{@field_split}]/)
+    value_split = Regexp::compile(@value_split_pattern || /[#{@value_split}]/)
+
+    optional_whitespace = /\s*/
+    eof = /$/
+
+    value_pattern = begin
+      # each component expression within value_pattern _must_ capture exactly once.
+      value_patterns = []
+
+      value_patterns << quoted_capture(%q(")) # quoted double
+      value_patterns << quoted_capture(%q(')) # quoted single
+      if @include_brackets
+        value_patterns << quoted_capture('(', ')') # bracketed paren
+        value_patterns << quoted_capture('[', ']') # bracketed square
+        value_patterns << quoted_capture('<', '>') # bracketed angle
+      end
+
+      # an unquoted value is a _captured_ sequence of characters or escaped spaces before a `field_split` or EOF.
+      value_patterns << /((?:\\ |.)+?)(?=#{Regexp::union(field_split, eof)})/
+
+      Regexp.union(*value_patterns)
+    end
+
+
+    # a key is a _captured_ sequence of characters or escaped spaces before optional whitespace
+    # and followed by either a `value_split`, a `field_split`, or EOF.
+    key_pattern = /((?:\\ |.)+?)(?=#{optional_whitespace}#{Regexp::union(value_split, field_split, eof)})/
+
+    @scan_re = /#{field_split}?#{key_pattern}#{optional_whitespace}(?:#{value_split}#{optional_whitespace}#{value_pattern})?(?=#{Regexp::union(field_split, eof)})/
+    @value_split_re = value_split
+
+    @logger.debug? && @logger.debug("KV scan regex", :regex => @scan_re.inspect)
   end
 
   def filter(event)
@@ -335,6 +408,26 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     s =~ @value_split_re
   end
 
+  # Helper function for generating single-capture `Regexp` that, when matching a string bound by the given quotes
+  # or brackets, will capture the content that is between the quotes or brackets.
+  #
+  # @api private
+  # @param quote_sequence [String] a character sequence that begins a quoted expression
+  # @param close_quote_sequence [String] a character sequence that ends a quoted expression; (default: quote_sequence)
+  # @return [Regexp] with a single capture group representing content that is between the given quotes
+  def quoted_capture(quote_sequence, close_quote_sequence=quote_sequence)
+    fail('quote_sequence must be non-empty!') if quote_sequence.nil? || quote_sequence.empty?
+    fail('close_quote_sequence must be non-empty!') if close_quote_sequence.nil? || close_quote_sequence.empty?
+
+    open_pattern = /#{Regexp.quote(quote_sequence)}/
+    close_pattern = /#{Regexp.quote(close_quote_sequence)}/
+
+    # matches a sequence of zero or more characters that is followed by the `close_quote_sequence`
+    quoted_value_pattern = /(?:.)*?(?=#{Regexp.quote(close_quote_sequence)})/
+
+    /#{open_pattern}(#{quoted_value_pattern})#{close_pattern}/
+  end
+
   def transform(text, method)
     case method
     when TRANSFORM_LOWERCASE_KEY
@@ -354,8 +447,10 @@ class LogStash::Filters::KV < LogStash::Filters::Base
     include_keys = @include_keys.map{|key| event.sprintf(key)}
     exclude_keys = @exclude_keys.map{|key| event.sprintf(key)}
 
-    text.scan(@scan_re) do |key, v1, v2, v3, v4, v5, v6|
-      value = v1 || v2 || v3 || v4 || v5 || v6
+    text.scan(@scan_re) do |key, *value_candidates|
+      value = value_candidates.compact.first
+      next if value.nil? || value.empty?
+
       key = @trim_key ? key.gsub(@trim_key_re, "") : key
       key = @remove_char_key ? key.gsub(@remove_char_key_re, "") : key
       key = @transform_key ? transform(key, @transform_key) : key

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '4.0.3'
+  s.version         = '4.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses key-value pairs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/kv_spec.rb

@@ -772,3 +772,129 @@ describe LogStash::Filters::KV do
     end
   end
 end
+
+describe "multi character splitting" do
+  subject do
+    plugin = LogStash::Filters::KV.new(options)
+    plugin.register
+    plugin
+  end
+
+  let(:data) { {"message" => message} }
+  let(:event) { LogStash::Event.new(data) }
+
+  shared_examples "parsing all fields and values" do
+    it "parses all fields and values" do
+      subject.filter(event)
+      expect(event.get("hello")).to eq("world")
+      expect(event.get("foo")).to eq("bar")
+      expect(event.get("baz")).to eq("fizz")
+      expect(event.get("doublequoted")).to eq("hello world")
+      expect(event.get("singlequoted")).to eq("hello world")
+      expect(event.get("bracketsone")).to eq("hello world")
+      expect(event.get("bracketstwo")).to eq("hello world")
+      expect(event.get("bracketsthree")).to eq("hello world")
+    end
+  end
+
+  context "empty value_split_pattern" do
+    let(:options) { { "value_split_pattern" => "" } }
+    it "should raise ConfigurationError" do
+      expect{subject}.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "empty field_split_pattern" do
+    let(:options) { { "field_split_pattern" => "" } }
+    it "should raise ConfigurationError" do
+      expect{subject}.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "single split" do
+    let(:message) { "hello:world foo:bar baz:fizz doublequoted:\"hello world\" singlequoted:'hello world' bracketsone:(hello world) bracketstwo:[hello world] bracketsthree:<hello world>" }
+    let(:options) {
+      {
+          "field_split" => " ",
+          "value_split" => ":",
+      }
+    }
+    it_behaves_like "parsing all fields and values"
+  end
+
+  context "value split multi" do
+    let(:message) { "hello::world foo::bar baz::fizz doublequoted::\"hello world\" singlequoted::'hello world' bracketsone::(hello world) bracketstwo::[hello world] bracketsthree::<hello world>" }
+    let(:options) {
+      {
+          "field_split" => " ",
+          "value_split_pattern" => "::",
+      }
+    }
+    it_behaves_like "parsing all fields and values"
+  end
+
+  context "field and value split multi" do
+    let(:message) { "hello::world__foo::bar__baz::fizz__doublequoted::\"hello world\"__singlequoted::'hello world'__bracketsone::(hello world)__bracketstwo::[hello world]__bracketsthree::<hello world>" }
+    let(:options) {
+      {
+          "field_split_pattern" => "__",
+          "value_split_pattern" => "::",
+      }
+    }
+    it_behaves_like "parsing all fields and values"
+  end
+
+  context "field and value split multi with regex" do
+    let(:message) { "hello:world_foo::bar__baz:::fizz___doublequoted:::\"hello world\"____singlequoted:::::'hello world'____bracketsone:::(hello world)__bracketstwo:[hello world]_bracketsthree::::::<hello world>" }
+    let(:options) {
+      {
+          "field_split_pattern" => "_+",
+          "value_split_pattern" => ":+",
+      }
+    }
+    it_behaves_like "parsing all fields and values"
+  end
+
+  context "field and value split multi using singe char" do
+    let(:message) { "hello:world foo:bar baz:fizz doublequoted:\"hello world\" singlequoted:'hello world' bracketsone:(hello world) bracketstwo:[hello world] bracketsthree:<hello world>" }
+    let(:options) {
+      {
+          "field_split_pattern" => " ",
+          "value_split_pattern" => ":",
+      }
+    }
+    it_behaves_like "parsing all fields and values"
+  end
+
+  context "field and value split multi using escaping" do
+    let(:message) { "hello++world??foo++bar??baz++fizz??doublequoted++\"hello world\"??singlequoted++'hello world'??bracketsone++(hello world)??bracketstwo++[hello world]??bracketsthree++<hello world>" }
+    let(:options) {
+      {
+          "field_split_pattern" => "\\?\\?",
+          "value_split_pattern" => "\\+\\+",
+      }
+    }
+    it_behaves_like "parsing all fields and values"
+  end
+
+
+  context "example from @guyboertje in #15" do
+    let(:message) { 'key1: val1; key2: val2; key3:  https://site/?g={......"...;  CLR  rv:11.0)"..}; key4: val4;' }
+    let(:options) {
+      {
+          "field_split_pattern" => ";\s*(?=key.+?:)|;$",
+          "value_split_pattern" => ":\s+",
+      }
+    }
+
+    it "parses all fields and values" do
+      subject.filter(event)
+
+      expect(event.get("key1")).to eq("val1")
+      expect(event.get("key2")).to eq("val2")
+      expect(event.get("key3")).to eq("https://site/?g={......\"...;  CLR  rv:11.0)\"..}")
+      expect(event.get("key4")).to eq("val4")
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 4.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Parses key-value pairs