logstash-filter-mutate 3.2.0 → 3.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8196cb41a7cc83ef6ff48ab4dc9be6f130a00048cfc16821ca93f62a00955fe
-  data.tar.gz: 7fccc907cd8ef3fc7c8c8f0b3ae91b7e61ebd7ff949a9265d3e35c0d33d92674
+  metadata.gz: e5423b0b68566ee643d458b6fb38c40a2b0fd336d9e2f23dd8921431adb1e7af
+  data.tar.gz: 470f2d561ea225dad08e4848146001fea9ba1eaba0f13ba652f00037703dd70e
 SHA512:
-  metadata.gz: 4ff52c37f5bc4525e97f73eaa605d646aae1a7474e0627c334ac0d3bfdc4d33736d201a3aaae0b4d0d6e7bf8ff1db3f08f3e1e8835719e5c33f15008c7746896
-  data.tar.gz: 7b2d637b7902aec46732838b327c307963da5ae279c1ff9ce91b5303adbf2a6411b75adc0944aec421226bd2c5e9efe2c0e059e79405c4e0d12d5b42e1a3dcd8
+  metadata.gz: 43b35a6bbe4a59e4999eec61169812bf917e17b09e41d158078b09fc476965c37c0d75c05eff5fc21238631c1bd56743d9ddf31395d70905090e7f648273d27e
+  data.tar.gz: f9e28cf3e4c431f8769106b9e856f9b85cb5218d5e713116354fc357947e561a2e0aebf9b054ff03a71204fb2b830e91705858dd43f2ff20de7704d2452a26ed

data/CHANGELOG.md

@@ -1,6 +1,12 @@
+## 3.3.1
+ - Fix: Number strings using a **decimal comma** (e.g. 1,23), added convert support to specify integer_eu and float_eu.
+
+## 3.3.0
+ - feature: Added capitalize feature.
+
 ## 3.2.0
   - Support boolean to integer conversion #107
-  
+
 ## 3.1.7
   - Update gemspec summary
 

data/CONTRIBUTORS

@@ -24,6 +24,7 @@ Contributors:
 * Suyog Rao (suyograo)
 * Tal Levy (talevy)
 * piavlo
+* Abdul Haseeb Hussain (AbdulHaseebHussain)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -37,12 +37,14 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-join>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-lowercase>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-merge>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-coerce>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-rename>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-replace>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-split>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-strip>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-update>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-uppercase>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-capitalize>> |<<array,array>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -51,7 +53,7 @@ filter plugins.
 &nbsp;
 
 [id="plugins-{type}s-{plugin}-convert"]
-===== `convert` 
+===== `convert`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -72,7 +74,13 @@ If the conversion type is `integer` and the value is a boolean, it will be conve
 * **True:**  `1`
 * **False:** `0`
 
-Valid conversion targets are: integer, float, string, and boolean.
+If you have numeric strings that have decimal commas (Europe and ex-colonies)
+e.g. "1.234,56" or "2.340", by using conversion targets of integer_eu or float_eu
+the convert function will treat "." as a group separator and "," as a decimal separator.
+
+Conversion targets of integer or float will now correctly handle "," as a group separator.
+
+Valid conversion targets are: integer, float, integer_eu, float_eu, string, and boolean.
 
 Example:
 [source,ruby]
@@ -83,7 +91,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-copy"]
-===== `copy` 
+===== `copy`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -99,7 +107,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-gsub"]
-===== `gsub` 
+===== `gsub`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -129,7 +137,7 @@ Example:
 
 
 [id="plugins-{type}s-{plugin}-join"]
-===== `join` 
+===== `join`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -145,7 +153,7 @@ Example:
    }
 
 [id="plugins-{type}s-{plugin}-lowercase"]
-===== `lowercase` 
+===== `lowercase`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -161,7 +169,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-merge"]
-===== `merge` 
+===== `merge`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -181,8 +189,25 @@ Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-coerce"]
+===== `coerce`
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Set the default value of a field that exists but is null
+
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        # Sets the default value of the 'field1' field to 'default_value'
+        coerce => { "field1" => "default_value" }
+      }
+    }
+
 [id="plugins-{type}s-{plugin}-rename"]
-===== `rename` 
+===== `rename`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -199,7 +224,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-replace"]
-===== `replace` 
+===== `replace`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -216,7 +241,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-split"]
-===== `split` 
+===== `split`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -233,7 +258,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-strip"]
-===== `strip` 
+===== `strip`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -249,7 +274,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-update"]
-===== `update` 
+===== `update`
 
   * Value type is <<hash,hash>>
   * There is no default value for this setting.
@@ -266,7 +291,7 @@ Example:
     }
 
 [id="plugins-{type}s-{plugin}-uppercase"]
-===== `uppercase` 
+===== `uppercase`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -281,7 +306,21 @@ Example:
       }
     }
 
+[id="plugins-{type}s-{plugin}-capitalize"]
+===== `capitalize`
+
+  * Value type is <<array,array>>
+  * There is no default value for this setting.
+
+Convert a string to its capitalized equivalent.
 
+Example:
+[source,ruby]
+    filter {
+      mutate {
+        capitalize => [ "fieldname" ]
+      }
+    }
 
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/mutate.rb

@@ -7,6 +7,18 @@ require "logstash/namespace"
 class LogStash::Filters::Mutate < LogStash::Filters::Base
   config_name "mutate"
 
+  # Sets a default value when the field exists but the value is null.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       mutate {
+  #         # Sets the default value of the 'field1' field to 'default_value'
+  #         coerce => { "field1" => "default_value" }
+  #       }
+  #     }
+  config :coerce, :validate => :hash
+
   # Rename one or more fields.
   #
   # Example:
@@ -59,7 +71,13 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
   # * **True:**  `1`
   # * **False:** `0`
   #
-  # Valid conversion targets are: integer, float, string, and boolean.
+  # If you have numeric strings that have decimal commas (Europe and ex-colonies)
+  # e.g. "1.234,56" or "2.340", by using conversion targets of integer_eu or float_eu
+  # the convert function will treat "." as a group separator and "," as a decimal separator.
+  #
+  # Conversion targets of integer or float will now correctly handle "," as a group separator.
+  #
+  # Valid conversion targets are: integer, float, integer_eu, float_eu, string, and boolean.
   #
   # Example:
   # [source,ruby]
@@ -117,6 +135,17 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
   #     }
   config :lowercase, :validate => :array
 
+  # Convert a string to its capitalized equivalent.
+  #
+  # Example:
+  # [source,ruby]
+  #     filter {
+  #       mutate {
+  #         capitalize => [ "fieldname" ]
+  #       }
+  #     }
+  config :capitalize, :validate => :array
+
   # Split a field to an array using a separator character. Only works on string
   # fields.
   #
@@ -183,7 +212,7 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
   CONVERT_PREFIX = "convert_".freeze
 
   def register
-    valid_conversions = %w(string integer float boolean)
+    valid_conversions = %w(string integer float boolean integer_eu float_eu )
     # TODO(sissel): Validate conversion requests if provided.
     @convert.nil? or @convert.each do |field, type|
       if !valid_conversions.include?(type)
@@ -216,12 +245,14 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
   end
 
   def filter(event)
+    coerce(event) if @coerce
     rename(event) if @rename
     update(event) if @update
     replace(event) if @replace
     convert(event) if @convert
     gsub(event) if @gsub
     uppercase(event) if @uppercase
+    capitalize(event) if @capitalize
     lowercase(event) if @lowercase
     strip(event) if @strip
     remove(event) if @remove
@@ -235,6 +266,13 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
 
   private
 
+  def coerce(event)
+    @coerce.each do |field, default_value|
+      next unless event.include?(field) && event.get(field)==nil
+      event.set(field, event.sprintf(default_value))
+    end
+  end
+
   def rename(event)
     @rename.each do |old, new|
       old = event.sprintf(old)
@@ -268,7 +306,7 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
       when Hash
         @logger.debug? && @logger.debug("I don't know how to type convert a hash, skipping", :field => field, :value => original)
       when Array
-        event.set(field, original.map { |v| converter.call(v) })
+        event.set(field, original.map { |v| v.nil? ? v : converter.call(v) })
       when NilClass
         # ignore
       else
@@ -288,21 +326,37 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
     value.to_s.force_encoding(Encoding::UTF_8)
   end
 
+  def convert_boolean(value)
+    return true if value =~ TRUE_REGEX
+    return false if value.empty? || value =~ FALSE_REGEX
+    @logger.warn("Failed to convert #{value} into boolean.")
+    value
+  end
+
   def convert_integer(value)
     return 1 if value == true
     return 0 if value == false
-    value.to_i
+    return value.to_i if !value.is_a?(String)
+    value.tr(",", "").to_i
   end
 
   def convert_float(value)
-    value.to_f
+    value.tr(",", "").to_f
   end
 
-  def convert_boolean(value)
-    return true if value =~ TRUE_REGEX
-    return false if value.empty? || value =~ FALSE_REGEX
-    @logger.warn("Failed to convert #{value} into boolean.")
-    value
+  def convert_integer_eu(value)
+    return 1 if value == true
+    return 0 if value == false
+    cnv_replace_eu(value).to_i
+  end
+
+  def convert_float_eu(value)
+    cnv_replace_eu(value).to_f
+  end
+
+  def cnv_replace_eu(value)
+    return value if !value.is_a?(String)
+    value.tr(".", "").tr(",", ".")
   end
 
   def gsub(event)
@@ -385,6 +439,26 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
     end
   end
 
+  def capitalize(event)
+    #see comments for #uppercase
+    @capitalize.each do |field|
+      original = event.get(field)
+      next if original.nil?
+      result = case original
+      when Array
+        original.map! do |elem|
+          (elem.is_a?(String) ? elem.capitalize : elem)
+        end
+      when String
+        original.capitalize! || original
+      else
+        @logger.debug? && @logger.debug("Can't capitalize something that isn't a string", :field => field, :value => original)
+        original
+      end
+      event.set(field, result)
+    end
+  end
+
   def split(event)
     @split.each do |field, separator|
       value = event.get(field)

data/logstash-filter-mutate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-mutate'
-  s.version         = '3.2.0'
+  s.version         = '3.3.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Performs mutations on fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/mutate_spec.rb

@@ -54,6 +54,24 @@ describe LogStash::Filters::Mutate do
     end
   end
 
+  context "when doing capitalize of an array" do
+
+    let(:config) do
+      { "capitalize" => ["array_of"] }
+    end
+
+    let(:attrs) { { "array_of" => ["ab", 2, "CDE"] } }
+
+    it "should capitalize not raise an error" do
+      expect { subject.filter(event) }.not_to raise_error
+    end
+
+    it "should convert only string elements" do
+      subject.filter(event)
+      expect(event.get("array_of")).to eq(["Ab", 2, "Cde"])
+    end
+  end
+
   context "when doing lowercase of an array" do
 
     let(:config) do
@@ -72,7 +90,7 @@ describe LogStash::Filters::Mutate do
     end
   end
 
-  %w(lowercase uppercase).each do |operation|
+  %w(lowercase uppercase capitalize).each do |operation|
     context "executing #{operation} a non-existant field" do
       let(:attrs) { }
 
@@ -262,12 +280,14 @@ describe LogStash::Filters::Mutate do
         mutate {
           lowercase => ["lowerme","Lowerme", "lowerMe"]
           uppercase => ["upperme", "Upperme", "upperMe"]
+          capitalize => ["capitalizeme", "Capitalizeme", "capitalizeMe"]
           convert => [ "intme", "integer", "floatme", "float" ]
           rename => [ "rename1", "rename2" ]
           replace => [ "replaceme", "hello world" ]
           replace => [ "newfield", "newnew" ]
           update => [ "nosuchfield", "weee" ]
           update => [ "updateme", "updated" ]
+
         }
       }
     CONFIG
@@ -275,10 +295,13 @@ describe LogStash::Filters::Mutate do
     event = {
       "lowerme" => "example",
       "upperme" => "EXAMPLE",
+      "capitalizeme" => "Example",
       "Lowerme" => "ExAmPlE",
       "Upperme" => "ExAmPlE",
+      "Capitalizeme" => "ExAmPlE",
       "lowerMe" => [ "ExAmPlE", "example" ],
       "upperMe" => [ "ExAmPlE", "EXAMPLE" ],
+      "capitalizeMe" => [ "ExAmPlE", "Example" ],
       "intme" => [ "1234", "7890.4", "7.9" ],
       "floatme" => [ "1234.455" ],
       "rename1" => [ "hello world" ],
@@ -289,10 +312,13 @@ describe LogStash::Filters::Mutate do
     sample event do
       expect(subject.get("lowerme")).to eq 'example'
       expect(subject.get("upperme")).to eq 'EXAMPLE'
+      expect(subject.get("capitalizeme")).to eq 'Example'
       expect(subject.get("Lowerme")).to eq 'example'
       expect(subject.get("Upperme")).to eq 'EXAMPLE'
+      expect(subject.get("Capitalizeme")).to eq 'Example'
       expect(subject.get("lowerMe")).to eq ['example', 'example']
       expect(subject.get("upperMe")).to eq ['EXAMPLE', 'EXAMPLE']
+      expect(subject.get("capitalizeMe")).to eq ['Example', 'Example']
       expect(subject.get("intme") ).to eq [1234, 7890, 7]
       expect(subject.get("floatme")).to eq [1234.455]
       expect(subject).not_to include("rename1")
@@ -311,6 +337,7 @@ describe LogStash::Filters::Mutate do
         mutate {
           lowercase => ["lowerme"]
           uppercase => ["upperme"]
+          capitalize => ["capitalizeme"]
         }
       }
     CONFIG
@@ -318,12 +345,14 @@ describe LogStash::Filters::Mutate do
     event = {
       "lowerme" => [ "АБВГД\0MMM", "こにちわ", "XyZółć", "NÎcË GÛŸ"],
       "upperme" => [ "аБвгд\0mmm", "こにちわ", "xYzółć", "Nîcë gûÿ"],
+      "capitalizeme" => ["АБВГД\0mmm", "こにちわ", "xyzółć", "nÎcË gÛŸ"],
     }
 
     sample event do
       # ATM, only the ASCII characters will case change
       expect(subject.get("lowerme")).to eq [ "АБВГД\0mmm", "こにちわ", "xyzółć", "nÎcË gÛŸ"]
       expect(subject.get("upperme")).to eq [ "аБвгд\0MMM", "こにちわ", "XYZółć", "NîCë Gûÿ"]
+      expect(subject.get("capitalizeme")).to eq [ "АБВГД\u0000mmm", "こにちわ", "Xyzółć", "NÎcË gÛŸ"]
     end
   end
 
@@ -520,7 +549,7 @@ describe LogStash::Filters::Mutate do
       expect(subject.get("[foo][bar]")).to eq 1000
       expect(subject.get("[foo][bar]")).to be_a(Fixnum)
     end
-  end
+ end
 
   describe "convert should work within arrays" do
     config <<-CONFIG
@@ -566,6 +595,102 @@ describe LogStash::Filters::Mutate do
     end
   end
 
+  describe "convert various US/UK strings" do
+    describe "to integer" do
+      config <<-CONFIG
+        filter {
+          mutate {
+            convert => {
+              "[foo][0]" => "integer"
+              "[foo][1]" => "integer"
+              "[foo][2]" => "integer"
+            }
+          }
+        }
+      CONFIG
+
+      sample({ "foo" => ["1,000", "1,234,567.8", "123.4"] }) do
+        expect(subject.get("[foo][0]")).to eq 1000
+        expect(subject.get("[foo][0]")).to be_a(Fixnum)
+        expect(subject.get("[foo][1]")).to eq 1234567
+        expect(subject.get("[foo][1]")).to be_a(Fixnum)
+        expect(subject.get("[foo][2]")).to eq 123
+        expect(subject.get("[foo][2]")).to be_a(Fixnum)
+      end
+    end
+
+    describe "to float" do
+      config <<-CONFIG
+        filter {
+          mutate {
+            convert => {
+              "[foo][0]" => "float"
+              "[foo][1]" => "float"
+              "[foo][2]" => "float"
+            }
+          }
+        }
+      CONFIG
+
+      sample({ "foo" => ["1,000", "1,234,567.8", "123.4"] }) do
+        expect(subject.get("[foo][0]")).to eq 1000.0
+        expect(subject.get("[foo][0]")).to be_a(Float)
+        expect(subject.get("[foo][1]")).to eq 1234567.8
+        expect(subject.get("[foo][1]")).to be_a(Float)
+        expect(subject.get("[foo][2]")).to eq 123.4
+        expect(subject.get("[foo][2]")).to be_a(Float)
+      end
+    end
+  end
+
+  describe "convert various EU style strings" do
+    describe "to integer" do
+      config <<-CONFIG
+        filter {
+          mutate {
+            convert => {
+              "[foo][0]" => "integer_eu"
+              "[foo][1]" => "integer_eu"
+              "[foo][2]" => "integer_eu"
+            }
+          }
+        }
+      CONFIG
+
+      sample({ "foo" => ["1.000", "1.234.567,8", "123,4"] }) do
+        expect(subject.get("[foo][0]")).to eq 1000
+        expect(subject.get("[foo][0]")).to be_a(Fixnum)
+        expect(subject.get("[foo][1]")).to eq 1234567
+        expect(subject.get("[foo][1]")).to be_a(Fixnum)
+        expect(subject.get("[foo][2]")).to eq 123
+        expect(subject.get("[foo][2]")).to be_a(Fixnum)
+      end
+    end
+
+    describe "to float" do
+      config <<-CONFIG
+        filter {
+          mutate {
+            convert => {
+              "[foo][0]" => "float_eu"
+              "[foo][1]" => "float_eu"
+              "[foo][2]" => "float_eu"
+            }
+          }
+        }
+      CONFIG
+
+      sample({ "foo" => ["1.000", "1.234.567,8", "123,4"] }) do
+        expect(subject.get("[foo][0]")).to eq 1000.0
+        expect(subject.get("[foo][0]")).to be_a(Float)
+        expect(subject.get("[foo][1]")).to eq 1234567.8
+        expect(subject.get("[foo][1]")).to be_a(Float)
+        expect(subject.get("[foo][2]")).to eq 123.4
+        expect(subject.get("[foo][2]")).to be_a(Float)
+      end
+    end
+  end
+
   #LOGSTASH-1529
   describe "gsub on a String with dynamic fields (%{}) in pattern" do
     config '
@@ -712,4 +837,26 @@ describe LogStash::Filters::Mutate do
     end
   end
 
+  describe "coerce arrays fields with default values when null" do
+    config '
+      filter {
+        mutate {
+          coerce => {
+            "field1" => "Hello"
+            "field2" => "Bye"
+            "field3" => 5
+            "field4" => false
+          }
+        }
+      }'
+
+
+    sample("field1" => nil, "field2" => nil, "field3" => nil, "field4" => true) do
+      expect(subject.get("field1")).to eq("Hello")
+      expect(subject.get("field2")).to eq("Bye")
+      expect(subject.get("field3")).to eq("5")
+      expect(subject.get("field4")).to eq(true)
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-mutate
 version: !ruby/object:Gem::Version
-  version: 3.2.0
+  version: 3.3.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-28 00:00:00.000000000 Z
+date: 2018-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement