logstash-filter-mutate 3.3.1 → 3.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5423b0b68566ee643d458b6fb38c40a2b0fd336d9e2f23dd8921431adb1e7af
-  data.tar.gz: 470f2d561ea225dad08e4848146001fea9ba1eaba0f13ba652f00037703dd70e
+  metadata.gz: 7e984a5018bec5903f3731c447c9e1fffe4b149c0c2e27ce9055e14f21deb9c0
+  data.tar.gz: 2b3c094ca15d06ac17c6746673ca31e40b7ca41b2217bd3c0f0ea8476c27f4c6
 SHA512:
-  metadata.gz: 43b35a6bbe4a59e4999eec61169812bf917e17b09e41d158078b09fc476965c37c0d75c05eff5fc21238631c1bd56743d9ddf31395d70905090e7f648273d27e
-  data.tar.gz: f9e28cf3e4c431f8769106b9e856f9b85cb5218d5e713116354fc357947e561a2e0aebf9b054ff03a71204fb2b830e91705858dd43f2ff20de7704d2452a26ed
+  metadata.gz: e7286e1ea17e30d46e942a7c8d1a9107c57a11d0158dc7bb979b1ac09c0b85a94159480b8b568b197c73d922369c90c0c85ca8e356e6648b501cc007c28a7fab
+  data.tar.gz: 7480daecb2a3c584302b6296597bc3d0f8583286ace082d6106daf6555c28705ecc73e6d28c0e8b92d41374fedf210f5cac156f5c1e431fa9f33f6709774272d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.3.2
+ - Fix: when converting to `float` and `float_eu`, explicitly support same range of inputs as their integer counterparts; eliminates a regression introduced in 3.3.1 in which support for non-string inputs was inadvertently removed.
+
 ## 3.3.1
  - Fix: Number strings using a **decimal comma** (e.g. 1,23), added convert support to specify integer_eu and float_eu.
 

data/docs/index.asciidoc

@@ -62,31 +62,38 @@ Convert a field's value to a different type, like turning a string to an
 integer. If the field value is an array, all members will be converted.
 If the field is a hash no action will be taken.
 
-If the conversion type is `boolean`, the acceptable values are:
-
-* **True:** `true`, `t`, `yes`, `y`, and `1`
-* **False:** `false`, `f`, `no`, `n`, and `0`
-
-If a value other than these is provided, it will pass straight through
-and log a warning message.
-
-If the conversion type is `integer` and the value is a boolean, it will be converted as:
-* **True:**  `1`
-* **False:** `0`
-
-If you have numeric strings that have decimal commas (Europe and ex-colonies)
-e.g. "1.234,56" or "2.340", by using conversion targets of integer_eu or float_eu
-the convert function will treat "." as a group separator and "," as a decimal separator.
-
-Conversion targets of integer or float will now correctly handle "," as a group separator.
-
-Valid conversion targets are: integer, float, integer_eu, float_eu, string, and boolean.
+Valid conversion targets, and their expected behaviour with different inputs are:
+
+ * `integer`:
+   - strings are parsed; comma-separators are supported (e.g., the string `"1,000"` produces an integer with value of one thousand); when strings have decimal parts, they are _truncated_.
+   - floats and decimals are _truncated_ (e.g., `3.99` becomes `3`, `-2.7` becomes `-2`)
+   - boolean true and boolean false are converted to `1` and `0` respectively
+ * `integer_eu`:
+   - same as `integer`, except string values support dot-separators and comma-decimals (e.g., `"1.000"` produces an integer with value of one thousand)
+ * `float`:
+   - integers are converted to floats
+   - strings are parsed; comma-separators and dot-decimals are supported (e.g., `"1,000.5"` produces an integer with value of one thousand and one half)
+   - boolean true and boolean false are converted to `1.0` and `0.0` respectively
+ * `float_eu`:
+   - same as `float`, except string values support dot-separators and comma-decimals (e.g., `"1.000,5"` produces an integer with value of one thousand and one half)
+ * `string`:
+   - all values are stringified and encoded with UTF-8
+ * `boolean`:
+   - strings `"true"`, `"t"`, `"yes"`, `"y"`, and `"1"` are converted to boolean `true`
+   - strings `"false"`, `"f"`, `"no"`, `"n"`, and `"0"` are converted to boolean `false`
+   - empty strings are converted to boolean `false`
+   - all other values pass straight through without conversion and log a warning message
+
+This plugin can convert multiple fields in the same document, see the example below.
 
 Example:
 [source,ruby]
     filter {
       mutate {
-        convert => { "fieldname" => "integer" }
+        convert => { 
+          "fieldname" => "integer"
+          "booleanfield" => "boolean"
+        }
       }
     }
 

data/lib/logstash/filters/mutate.rb

@@ -341,22 +341,29 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
   end
 
   def convert_float(value)
-    value.tr(",", "").to_f
+    return 1.0 if value == true
+    return 0.0 if value == false
+    value = value.delete(",") if value.kind_of?(String)
+    value.to_f
   end
 
   def convert_integer_eu(value)
-    return 1 if value == true
-    return 0 if value == false
-    cnv_replace_eu(value).to_i
+    us_value = cnv_replace_eu(value)
+    convert_integer(us_value)
   end
 
   def convert_float_eu(value)
-    cnv_replace_eu(value).to_f
+    us_value = cnv_replace_eu(value)
+    convert_float(us_value)
   end
 
+  # When given a String, returns a new String whose contents have been converted from
+  # EU-style comma-decimals and dot-separators to US-style dot-decimals and comma-separators.
+  #
+  # For all other values, returns value unmodified.
   def cnv_replace_eu(value)
     return value if !value.is_a?(String)
-    value.tr(".", "").tr(",", ".")
+    value.tr(",.", ".,")
   end
 
   def gsub(event)

data/logstash-filter-mutate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-mutate'
-  s.version         = '3.3.1'
+  s.version         = '3.3.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Performs mutations on fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/mutate_spec.rb

@@ -415,6 +415,152 @@ describe LogStash::Filters::Mutate do
     end
   end
 
+  describe "convert to float" do
+
+    config <<-CONFIG
+      filter {
+        mutate {
+          convert => {
+            "field" => "float"
+          }
+        }
+      }
+    CONFIG
+
+    context 'when field is a string with no separator and dot decimal' do
+      sample({'field' => '3141.5926'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.5926)
+      end
+    end
+
+    context 'when field is a string with a comma separator and dot decimal' do
+      sample({'field' => '3,141.5926'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.5926)
+      end
+    end
+
+    context 'when field is a string comma separator and no decimal' do
+      sample({'field' => '3,141'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.0)
+      end
+    end
+
+    context 'when field is a string no separator and no decimal' do
+      sample({'field' => '3141'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.0)
+      end
+    end
+
+    context 'when field is a float' do
+      sample({'field' => 3.1415926}) do
+        expect(subject.get('field')).to be_within(0.000001).of(3.1415926)
+      end
+    end
+
+    context 'when field is an integer' do
+      sample({'field' => 3}) do
+        expect(subject.get('field')).to be_within(0.000001).of(3)
+      end
+    end
+
+    context 'when field is the true value' do
+      sample('field' => true) do
+        expect(subject.get('field')).to eq(1.0)
+      end
+    end
+
+    context 'when field is the false value' do
+      sample('field' => false) do
+        expect(subject.get('field')).to eq(0.0)
+      end
+    end
+
+    context 'when field is nil' do
+      sample('field' => nil) do
+        expect(subject.get('field')).to be_nil
+      end
+    end
+
+    context 'when field is not set' do
+      sample('field' => nil) do
+        expect(subject.get('field')).to be_nil
+      end
+    end
+  end
+
+
+  describe "convert to float_eu" do
+    config <<-CONFIG
+      filter {
+        mutate {
+          convert => {
+            "field" => "float_eu"
+          }
+        }
+      }
+    CONFIG
+
+    context 'when field is a string with no separator and comma decimal' do
+      sample({'field' => '3141,5926'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.5926)
+      end
+    end
+
+    context 'when field is a string with a dot separator and comma decimal' do
+      sample({'field' => '3.141,5926'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.5926)
+      end
+    end
+
+    context 'when field is a string dot separator and no decimal' do
+      sample({'field' => '3.141'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.0)
+      end
+    end
+
+    context 'when field is a string no separator and no decimal' do
+      sample({'field' => '3141'}) do
+        expect(subject.get('field')).to be_within(0.0001).of(3141.0)
+      end
+    end
+
+    context 'when field is a float' do
+      sample({'field' => 3.1415926}) do
+        expect(subject.get('field')).to be_within(0.000001).of(3.1415926)
+      end
+    end
+
+    context 'when field is an integer' do
+      sample({'field' => 3}) do
+        expect(subject.get('field')).to be_within(0.000001).of(3)
+      end
+    end
+
+    context 'when field is the true value' do
+      sample('field' => true) do
+        expect(subject.get('field')).to eq(1.0)
+      end
+    end
+
+    context 'when field is the false value' do
+      sample('field' => false) do
+        expect(subject.get('field')).to eq(0.0)
+      end
+    end
+
+    context 'when field is nil' do
+      sample('field' => nil) do
+        expect(subject.get('field')).to be_nil
+      end
+    end
+
+    context 'when field is not set' do
+      sample('field' => nil) do
+        expect(subject.get('field')).to be_nil
+      end
+    end
+  end
+
   describe "gsub on a String" do
     config '
       filter {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-mutate
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.3.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-02 00:00:00.000000000 Z
+date: 2018-05-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement