logstash-filter-kv 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd187e13f268bccf3d773032be91182db9539218
-  data.tar.gz: 3869c2cd4527da8298067f4558d809b7c034e87d
+  metadata.gz: 010ddb862a2bcdf8e8a0eb272b1b6edbb9710169
+  data.tar.gz: bfa43c0dcf9e9978f1fb8cc71cc6712f16c0c503
 SHA512:
-  metadata.gz: f6d25cd2beb66fad753125e9f0f49806dbf908771ab7ef1ec73584feb5538b36857cec4ba5e6f75df30aecab87d5ee2b573d6c60b9b22096a3cc27484859e42f
-  data.tar.gz: 905f6b2f0fc3ab89dfe95a3cfbd2b5e76a7f50835e0722eea69e94b4e01b387386252dacb76200c512d2390d1579647d6863c8138f5e2ee43d3bbb1cd9112552
+  metadata.gz: 6da0c526856467869d33fa7878840af0d41543c27918c053cec520714d1b24dd141a1ccc01c8eb8d413b1a7c22e9754ba6854c38fc46f931570bc35db54ba342
+  data.tar.gz: 01a782a29a6a9310eb0f70a1ff0c980c131e5a80cb8a3796ffd563cef169d35dc8f58cc1216f5156acd2959a0afddbd650257a24e863a1a7fc51f91448104224

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,409 @@
+:plugin: kv
+:type: filter
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Kv filter plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+This filter helps automatically parse messages (or specific event fields)
+which are of the `foo=bar` variety.
+
+For example, if you have a log message which contains `ip=1.2.3.4
+error=REFUSED`, you can parse those automatically by configuring:
+[source,ruby]
+    filter {
+      kv { }
+    }
+
+The above will result in a message of `ip=1.2.3.4 error=REFUSED` having
+the fields:
+
+* `ip: 1.2.3.4`
+* `error: REFUSED`
+
+This is great for postfix, iptables, and other types of logs that
+tend towards `key=value` syntax.
+
+You can configure any arbitrary strings to split your data on,
+in case your data is not structured using `=` signs and whitespace.
+For example, this filter can also be used to parse query parameters like
+`foo=bar&baz=fizz` by setting the `field_split` parameter to `&`.
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Kv Filter Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-allow_duplicate_values>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-default_keys>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-exclude_keys>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-field_split>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-include_brackets>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-include_keys>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-prefix>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-recursive>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-remove_char_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-remove_char_value>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-source>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-transform_key>> |<<string,string>>, one of `["lowercase", "uppercase", "capitalize"]`|No
+| <<plugins-{type}s-{plugin}-transform_value>> |<<string,string>>, one of `["lowercase", "uppercase", "capitalize"]`|No
+| <<plugins-{type}s-{plugin}-trim_key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-trim_value>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-value_split>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+filter plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-allow_duplicate_values"]
+===== `allow_duplicate_values` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+A bool option for removing duplicate key/value pairs. When set to false, only
+one unique key/value pair will be preserved.
+
+For example, consider a source like `from=me from=me`. `[from]` will map to
+an Array with two elements: `["me", "me"]`. To only keep unique key/value pairs,
+you could use this configuration:
+[source,ruby]
+    filter {
+      kv {
+        allow_duplicate_values => false
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-default_keys"]
+===== `default_keys` 
+
+  * Value type is <<hash,hash>>
+  * Default value is `{}`
+
+A hash specifying the default keys and their values which should be added to the event
+in case these keys do not exist in the source field being parsed.
+[source,ruby]
+    filter {
+      kv {
+        default_keys => [ "from", "logstash@example.com",
+                         "to", "default@dev.null" ]
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-exclude_keys"]
+===== `exclude_keys` 
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+An array specifying the parsed keys which should not be added to the event.
+By default no keys will be excluded.
+
+For example, consider a source like `Hey, from=<abc>, to=def foo=bar`.
+To exclude `from` and `to`, but retain the `foo` key, you could use this configuration:
+[source,ruby]
+    filter {
+      kv {
+        exclude_keys => [ "from", "to" ]
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-field_split"]
+===== `field_split` 
+
+  * Value type is <<string,string>>
+  * Default value is `" "`
+
+A string of characters to use as delimiters for parsing out key-value pairs.
+
+These characters form a regex character class and thus you must escape special regex
+characters like `[` or `]` using `\`.
+
+#### Example with URL Query Strings
+
+For example, to split out the args from a url query string such as
+`?pin=12345~0&d=123&e=foo@bar.com&oq=bobo&ss=12345`:
+[source,ruby]
+    filter {
+      kv {
+        field_split => "&?"
+      }
+    }
+
+The above splits on both `&` and `?` characters, giving you the following
+fields:
+
+* `pin: 12345~0`
+* `d: 123`
+* `e: foo@bar.com`
+* `oq: bobo`
+* `ss: 12345`
+
+[id="plugins-{type}s-{plugin}-include_brackets"]
+===== `include_brackets` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+A boolean specifying whether to treat square brackets, angle brackets,
+and parentheses as value "wrappers" that should be removed from the value.
+[source,ruby]
+    filter {
+      kv {
+        include_brackets => true
+      }
+    }
+
+For example, the result of this line:
+`bracketsone=(hello world) bracketstwo=[hello world] bracketsthree=<hello world>`
+
+will be:
+
+* bracketsone: hello world
+* bracketstwo: hello world
+* bracketsthree: hello world
+
+instead of:
+
+* bracketsone: (hello
+* bracketstwo: [hello
+* bracketsthree: <hello
+
+
+[id="plugins-{type}s-{plugin}-include_keys"]
+===== `include_keys` 
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+An array specifying the parsed keys which should be added to the event.
+By default all keys will be added.
+
+For example, consider a source like `Hey, from=<abc>, to=def foo=bar`.
+To include `from` and `to`, but exclude the `foo` key, you could use this configuration:
+[source,ruby]
+    filter {
+      kv {
+        include_keys => [ "from", "to" ]
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-prefix"]
+===== `prefix` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+A string to prepend to all of the extracted keys.
+
+For example, to prepend arg_ to all keys:
+[source,ruby]
+    filter { kv { prefix => "arg_" } }
+
+[id="plugins-{type}s-{plugin}-recursive"]
+===== `recursive` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+A boolean specifying whether to drill down into values
+and recursively get more key-value pairs from it.
+The extra key-value pairs will be stored as subkeys of the root key.
+
+Default is not to recursive values.
+[source,ruby]
+    filter {
+      kv {
+        recursive => "true"
+      }
+    }
+
+
+[id="plugins-{type}s-{plugin}-remove_char_key"]
+===== `remove_char_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A string of characters to remove from the key.
+
+These characters form a regex character class and thus you must escape special regex
+characters like `[` or `]` using `\`.
+
+Contrary to trim option, all characters are removed from the key, whatever their position.
+
+For example, to remove `<` `>` `[` `]` and `,` characters from keys:
+[source,ruby]
+    filter {
+      kv {
+        remove_char_key => "<>\[\],"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-remove_char_value"]
+===== `remove_char_value` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A string of characters to remove from the value.
+
+These characters form a regex character class and thus you must escape special regex
+characters like `[` or `]` using `\`.
+
+Contrary to trim option, all characters are removed from the value, whatever their position.
+
+For example, to remove `<`, `>`, `[`, `]` and `,` characters from values:
+[source,ruby]
+    filter {
+      kv {
+        remove_char_value => "<>\[\],"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-source"]
+===== `source` 
+
+  * Value type is <<string,string>>
+  * Default value is `"message"`
+
+The field to perform `key=value` searching on
+
+For example, to process the `not_the_message` field:
+[source,ruby]
+    filter { kv { source => "not_the_message" } }
+
+[id="plugins-{type}s-{plugin}-target"]
+===== `target` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The name of the container to put all of the key-value pairs into.
+
+If this setting is omitted, fields will be written to the root of the
+event, as individual fields.
+
+For example, to place all keys into the event field kv:
+[source,ruby]
+    filter { kv { target => "kv" } }
+
+[id="plugins-{type}s-{plugin}-transform_key"]
+===== `transform_key` 
+
+  * Value can be any of: `lowercase`, `uppercase`, `capitalize`
+  * There is no default value for this setting.
+
+Transform keys to lower case, upper case or capitals.
+
+For example, to lowercase all keys:
+[source,ruby]
+    filter {
+      kv {
+        transform_key => "lowercase"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-transform_value"]
+===== `transform_value` 
+
+  * Value can be any of: `lowercase`, `uppercase`, `capitalize`
+  * There is no default value for this setting.
+
+Transform values to lower case, upper case or capitals.
+
+For example, to capitalize all values:
+[source,ruby]
+    filter {
+      kv {
+        transform_value => "capitalize"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-trim_key"]
+===== `trim_key` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A string of characters to trim from the key. This is useful if your
+keys are wrapped in brackets or start with space.
+
+These characters form a regex character class and thus you must escape special regex
+characters like `[` or `]` using `\`.
+
+Only leading and trailing characters are trimed from the key.
+
+For example, to trim `<` `>` `[` `]` and `,` characters from keys:
+[source,ruby]
+    filter {
+      kv {
+        trim_key => "<>\[\],"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-trim_value"]
+===== `trim_value` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Constants used for transform check
+A string of characters to trim from the value. This is useful if your
+values are wrapped in brackets or are terminated with commas (like postfix
+logs).
+
+These characters form a regex character class and thus you must escape special regex
+characters like `[` or `]` using `\`.
+
+Only leading and trailing characters are trimed from the value.
+
+For example, to trim `<`, `>`, `[`, `]` and `,` characters from values:
+[source,ruby]
+    filter {
+      kv {
+        trim_value => "<>\[\],"
+      }
+    }
+
+[id="plugins-{type}s-{plugin}-value_split"]
+===== `value_split` 
+
+  * Value type is <<string,string>>
+  * Default value is `"="`
+
+A non-empty string of characters to use as delimiters for identifying key-value relations.
+
+These characters form a regex character class and thus you must escape special regex
+characters like `[` or `]` using `\`.
+
+For example, to identify key-values such as
+`key1:value1 key2:value2`:
+[source,ruby]
+    filter { kv { value_split => ":" } }
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-filter-kv.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-kv'
-  s.version         = '4.0.0'
+  s.version         = '4.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter helps automatically parse messages (or specific event fields) which are of the 'foo=bar' variety."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/filters/kv_spec.rb

@@ -669,7 +669,16 @@ describe LogStash::Filters::KV do
       plugin
     end
 
-    let(:message) { "AccountStatus: 4\r\nAdditionalInformation\r\n\r\nCode: \r\nHttpStatusCode: \r\nIsSuccess: True\r\nMessage: \r\n" }
+    let(:f1) { "AccountStatus" }
+    let(:v1) { "4" }
+    let(:f2) { "AdditionalInformation" }
+    let(:f3) { "Code" }
+    let(:f4) { "HttpStatusCode" }
+    let(:f5) { "IsSuccess" }
+    let(:v5) { "True" }
+    let(:f6) { "Message" }
+
+    let(:message) { "#{f1}: #{v1}\r\n#{f2}\r\n\r\n#{f3}: \r\n#{f4}: \r\n#{f5}: #{v5}\r\n#{f6}: \r\n" }
     let(:data) { {"message" => message} }
     let(:event) { LogStash::Event.new(data) }
     let(:options) {
@@ -683,10 +692,12 @@ describe LogStash::Filters::KV do
     context "key and splitters with no value" do
       it "should ignore the incomplete key/value pairs" do
         subject.filter(event)
-        expect(event.get("AccountStatus")).to eq("4")
-        expect(event.get("IsSuccess")).to eq("True")
-        expect(event.to_hash.keys.sort).to eq(
-          ["@timestamp", "@version", "AccountStatus", "IsSuccess", "message", "tags"])
+        expect(event.get(f1)).to eq(v1)
+        expect(event.get(f5)).to eq(v5)
+        expect(event.include?(f2)).to be false
+        expect(event.include?(f3)).to be false
+        expect(event.include?(f4)).to be false
+        expect(event.include?(f6)).to be false
       end
     end
   end
@@ -715,8 +726,6 @@ describe LogStash::Filters::KV do
         subject.filter(event)
         expect(event.get("key1")).to eq("value1 with spaces")
         expect(event.get("key2 with spaces")).to eq("value2")
-        expect(event.to_hash.keys.sort).to eq(
-          ["@timestamp", "@version", "key1", "key2 with spaces", "message", "tags"])
       end
     end
   end
@@ -745,8 +754,6 @@ describe LogStash::Filters::KV do
         subject.filter(event)
         expect(event.get("key1")).to eq("value1withspaces")
         expect(event.get("key2withspaces")).to eq("value2")
-        expect(event.to_hash.keys.sort).to eq(
-          ["@timestamp", "@version", "key1", "key2withspaces", "message", "tags"])
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-kv
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-06 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -56,6 +56,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/filters/kv.rb
 - logstash-filter-kv.gemspec
 - spec/filters/kv_spec.rb