logstash-filter-translate 3.2.2 → 3.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c2b0e57299901e68b342a5f9bc038c52a37a214d47302dd3f05718b1c954968
-  data.tar.gz: a946264b317825a0477a1d547faf3843826d7f391eb13d117a5060e9285e8d1f
+  metadata.gz: 21079c87f27ea96b5aafd79d5a48405a22fdbf1e7ad14de01a4f5f79b064b21f
+  data.tar.gz: b0c6f572eaf3d2dc53887838d1947ab5a8aac11a8718dc9a1f5f4fbe695fcc48
 SHA512:
-  metadata.gz: 3f4983b4527a2de1d00682269b374e2828caccdc3f3f1a57bd560d3b098b65a6475c2fb0f6288792805bb0fab2144f95df9dc1c280b228064196eaa2fc30e3b0
-  data.tar.gz: 1c2e0ead540188084abcc23033080e19d009b6651b757cefb6bd1c5ace26801df5d996296f9ddfba9de0d0a0b7a5cb2f7ac672d5cf2b999dc3e45b90f24b809c
+  metadata.gz: 53c86c7099d90e31007f68f83e29f5867d1f81e162b0fe01fdde8ca07d3e3a063250586fff399193cee50366b360da60602711c38515ca4b7722eb36edcf6c1d
+  data.tar.gz: 1d4db2634a1d862bce62fdd3dd150b5985a657cdc78d7f2b90426dd5765879aa1a654ad30aec7f43bf93abd3edd220333002dcd03b42a971a77835a8a4faed59

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 3.2.3
+  - Fix to align with docs - looked-up values are always strings. Coerce better. [#77](https://github.com/logstash-plugins/logstash-filter-translate/pull/77)
+  - Fix bug in dictionary/file the always applied RegexExact, manifested when dictionary keys are not regex compatible [Logstash #9936](https://github.com/elastic/logstash/issues/9936)
+  - Added info to dictionary_path description to explain why integers must be quoted
+
 ## 3.2.2
   - Fix bug in csv_file when LS config has CSV filter plugin specified as well as a csv dictionary.
   [#70](https://github.com/logstash-plugins/logstash-filter-translate/issues/70)

data/docs/index.asciidoc

@@ -24,11 +24,14 @@ A general search and replace tool that uses a configured hash
 and/or a file to determine replacement values. Currently supported are
 YAML, JSON, and CSV files. Each dictionary item is a key value pair.
 
-The dictionary entries can be specified in one of two ways: First,
-the `dictionary` configuration item may contain a hash representing
-the mapping. Second, an external file (readable by logstash) may be specified
-in the `dictionary_path` configuration item. These two methods may not be used
-in conjunction; it will produce an error.
+You can specify dictionary entries in one of two ways: 
+
+* The `dictionary` configuration item can contain a hash representing
+the mapping. 
+* An external file (readable by logstash) may be specified in the
+`dictionary_path` configuration item. 
+
+These two methods may not be used in conjunction; it will produce an error.
 
 Operationally, for each event, the value from the `field` setting is tested
 against the dictionary and if it matches exactly (or matches a regex when
@@ -146,9 +149,14 @@ NOTE: It is an error to specify both `dictionary` and `dictionary_path`.
   * Value type is <<path,path>>
   * There is no default value for this setting.
 
-The full path of the external dictionary file. The format of the table
-should be a standard YAML, JSON, or CSV. Make sure you specify any integer-based keys
-in quotes. For example, the YAML file should look something like this:
+The full path of the external dictionary file. The format of the table should be
+a standard YAML, JSON, or CSV. 
+
+Specify any integer-based keys in quotes. The
+value taken from the event's `field` setting is converted to a string. The
+lookup dictionary keys must also be strings, and the quotes make the
+integer-based keys function as a string. For example, the YAML file should look
+something like this:
 
 [source,ruby]
 ----
@@ -354,8 +362,10 @@ A value of zero or less will disable refresh.
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-If you'd like to treat dictionary keys as regular expressions, set `regex => true`.
-Note: this is activated only when `exact => true`.
+To treat dictionary keys as regular expressions, set `regex => true`.
+
+Be sure to escape dictionary key strings for use with regex.
+Resources on regex formatting are available online.
 
 [id="plugins-{type}s-{plugin}-refresh_behaviour"]
 ===== `refresh_behaviour`

data/lib/logstash/filters/array_of_maps_value_update.rb

@@ -25,7 +25,7 @@ module LogStash module Filters
         inner = event.get(nested_field)
         next if inner.nil?
         matched = [true, nil]
-        @lookup.fetch_strategy.fetch(inner, matched)
+        @lookup.fetch_strategy.fetch(inner.to_s, matched)
         if matched.first
           event.set(nested_destination, matched.last)
           matches[index] = true

data/lib/logstash/filters/array_of_values_update.rb

@@ -2,30 +2,40 @@
 
 module LogStash module Filters
   class ArrayOfValuesUpdate
+    class CoerceArray
+      def call(source) source; end
+    end
+    class CoerceOther
+      def call(source) Array(source); end
+    end
+
     def initialize(iterate_on, destination, fallback, lookup)
       @iterate_on = iterate_on
       @destination = destination
       @fallback = fallback
       @use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
       @lookup = lookup
+      @coercers_table = {}
+      @coercers_table.default = CoerceOther.new
+      @coercers_table[Array] = CoerceArray.new
     end
 
     def test_for_inclusion(event, override)
       # Skip translation in case @destination iterate_on already exists and @override is disabled.
-      return false if event.include?(@destination) && !override
+      return false if !override && event.include?(@destination)
       event.include?(@iterate_on)
     end
 
     def update(event)
       val = event.get(@iterate_on)
-      source = Array(val)
+      source = @coercers_table[val.class].call(val)
       target = Array.new(source.size)
       if @use_fallback
         target.fill(event.sprintf(@fallback))
       end
       source.each_with_index do |inner, index|
         matched = [true, nil]
-        @lookup.fetch_strategy.fetch(inner, matched)
+        @lookup.fetch_strategy.fetch(inner.to_s, matched)
         if matched.first
           target[index] = matched.last
         end

data/lib/logstash/filters/dictionary/file.rb

@@ -44,11 +44,12 @@ module LogStash module Filters module Dictionary
       @update_method = method(:merge_dictionary)
       initialize_for_file_type
       args = [@dictionary, rw_lock]
-      if exact
-        @fetch_strategy = regex ? FetchStrategy::File::ExactRegex.new(*args) : FetchStrategy::File::ExactRegex.new(*args)
-      else
-        @fetch_strategy = FetchStrategy::File::RegexUnion.new(*args)
-      end
+      klass = case
+              when exact && regex then FetchStrategy::File::ExactRegex
+              when exact          then FetchStrategy::File::Exact
+              else                     FetchStrategy::File::RegexUnion
+              end
+      @fetch_strategy = klass.new(*args)
       load_dictionary(raise_exception = true)
       stop_scheduler(initial = true)
       start_scheduler unless @refresh_interval <= 0 # disabled, a scheduler interval of zero makes no sense
@@ -131,7 +132,9 @@ module LogStash module Filters module Dictionary
     def loading_exception(e, raise_exception)
       msg = "Translate: #{e.message} when loading dictionary file at #{@dictionary_path}"
       if raise_exception
-        raise DictionaryFileError.new(msg)
+        dfe = DictionaryFileError.new(msg)
+        dfe.set_backtrace(e.backtrace)
+        raise dfe
       else
         @logger.warn("#{msg}, continuing with old dictionary", :dictionary_path => @dictionary_path)
       end

data/lib/logstash/filters/dictionary/memory.rb

@@ -7,11 +7,12 @@ module LogStash module Filters module Dictionary
     attr_reader :dictionary, :fetch_strategy
 
     def initialize(hash, exact, regex)
-      if exact
-        @fetch_strategy = regex ? FetchStrategy::Memory::ExactRegex.new(hash) : FetchStrategy::Memory::Exact.new(hash)
-      else
-        @fetch_strategy = FetchStrategy::Memory::RegexUnion.new(hash)
-      end
+      klass = case
+              when exact && regex then FetchStrategy::Memory::ExactRegex
+              when exact          then FetchStrategy::Memory::Exact
+              else                     FetchStrategy::Memory::RegexUnion
+              end
+      @fetch_strategy = klass.new(hash)
     end
 
     def stop_scheduler

data/lib/logstash/filters/single_value_update.rb

@@ -2,23 +2,39 @@
 
 module LogStash module Filters
   class SingleValueUpdate
+    class CoerceString
+      def call(source) source; end
+    end
+    class CoerceArray
+      def call(source) source.first.to_s; end
+    end
+    class CoerceOther
+      def call(source) source.to_s end
+    end
+
     def initialize(field, destination, fallback, lookup)
       @field = field
       @destination = destination
       @fallback = fallback
       @use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
       @lookup = lookup
+      @coercers_table = {}
+      @coercers_table.default = CoerceOther.new
+      @coercers_table[String] = CoerceString.new
+      @coercers_table[Array] = CoerceArray.new
     end
 
     def test_for_inclusion(event, override)
       # Skip translation in case @destination field already exists and @override is disabled.
-      return false if event.include?(@destination) && !override
+      return false if !override && event.include?(@destination)
       event.include?(@field)
     end
 
     def update(event)
       # If source field is array use first value and make sure source value is string
-      source = Array(event.get(@field)).first.to_s
+      # source = Array(event.get(@field)).first.to_s
+      source = event.get(@field)
+      source = @coercers_table[source.class].call(source)
       matched = [true, nil]
       @lookup.fetch_strategy.fetch(source, matched)
       if matched.first

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.2.2'
+  s.version         = '3.2.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/translate_spec.rb

@@ -2,6 +2,12 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/translate"
 
+module TranslateUtil
+  def self.build_fixture_path(filename)
+    File.join(File.dirname(__FILE__), "..", "fixtures", filename)
+  end
+end
+
 describe LogStash::Filters::Translate do
 
   let(:config) { Hash.new }
@@ -24,13 +30,36 @@ describe LogStash::Filters::Translate do
 
     let(:event) { LogStash::Event.new("status" => 200) }
 
-    it "return the exact translation" do
+    it "coerces field to a string then returns the exact translation" do
       subject.register
       subject.filter(event)
       expect(event.get("translation")).to eq("OK")
     end
   end
 
+  describe "translation fails when regex setting is false but keys are regex based" do
+
+    let(:config) do
+      {
+        "field"       => "status",
+        "destination" => "translation",
+        "dictionary"  => [ "^2\\d\\d", "OK",
+                           "^3\\d\\d", "Redirect",
+                           "^4\\d\\d", "Client Error",
+                           "^5\\d\\d", "Server Error" ],
+        "exact"       => true,
+        "regex"       => false
+      }
+    end
+
+    let(:event) { LogStash::Event.new("status" => 200) }
+
+    it "does not return the exact translation" do
+      subject.register
+      subject.filter(event)
+      expect(event.get("translation")).to be_nil
+    end
+  end
 
   describe "multi translation" do
     context "when using an inline dictionary" do
@@ -57,7 +86,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a file based dictionary" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "regex_union_dict.csv") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
       let(:config) do
         {
           "field"       => "status",
@@ -104,7 +133,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a file based dictionary" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "regex_dict.csv") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_dict.csv") }
       let(:config) do
         {
           "field"       => "status",
@@ -167,7 +196,7 @@ describe LogStash::Filters::Translate do
 
   describe "loading a dictionary" do
 
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict-wrong.yml") }
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict-wrong.yml") }
 
     let(:config) do
       {
@@ -186,7 +215,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a yml file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
       let(:event) { LogStash::Event.new("status" => "a") }
 
       it "return the exact translation" do
@@ -197,7 +226,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a map tagged yml file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
       let(:event) { LogStash::Event.new("status" => "six") }
 
       it "return the exact translation" do
@@ -208,7 +237,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a omap tagged yml file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-omap-dict.yml") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-omap-dict.yml") }
       let(:event) { LogStash::Event.new("status" => "nine") }
 
       it "return the exact translation" do
@@ -219,7 +248,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a json file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.json") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.json") }
       let(:event) { LogStash::Event.new("status" => "b") }
 
       it "return the exact translation" do
@@ -230,7 +259,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a csv file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.csv") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.csv") }
       let(:event) { LogStash::Event.new("status" => "c") }
 
       it "return the exact translation" do
@@ -241,7 +270,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using an unknown file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.other") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.other") }
 
       it "raises error" do
         expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} has a non valid format/)
@@ -250,20 +279,21 @@ describe LogStash::Filters::Translate do
   end
 
   describe "iterate_on functionality" do
-    describe "when iterate_on is the same as field, AKA array of values" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
-      let(:config) do
-        {
-          "iterate_on"          => "foo",
-          "field"            => "foo",
-          "destination"      => "baz",
-          "fallback"         => "nooo",
-          "dictionary_path"  => dictionary_path,
-          # "override"         => true,
-          "refresh_interval" => 0
-        }
-      end
+    let(:config) do
+      {
+        "iterate_on"       => "foo",
+        "field"            => iterate_on_field,
+        "destination"      => "baz",
+        "fallback"         => "nooo",
+        "dictionary_path"  => dictionary_path,
+        # "override"         => true,
+        "refresh_interval" => 0
+      }
+    end
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
 
+    describe "when iterate_on is the same as field, AKA array of values" do
+      let(:iterate_on_field) { "foo" }
       let(:event) { LogStash::Event.new("foo" => ["nine","eight", "seven"]) }
       it "adds a translation to destination array for each value in field array" do
         subject.register
@@ -272,20 +302,19 @@ describe LogStash::Filters::Translate do
       end
     end
 
-    describe "when iterate_on is not the same as field, AKA array of objects" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
-      let(:config) do
-        {
-          "iterate_on"          => "foo",
-          "field"            => "bar",
-          "destination"      => "baz",
-          "fallback"         => "nooo",
-          "dictionary_path"  => dictionary_path,
-          # "override"         => true,
-          "refresh_interval" => 0
-        }
+    describe "when iterate_on is the same as field, AKA array of values, coerces integer elements to strings" do
+      let(:iterate_on_field) { "foo" }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
+      let(:event) { LogStash::Event.new("foo" => [200, 300, 400]) }
+      it "adds a translation to destination array for each value in field array" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("baz")).to eq(["OK","Redirect","Client Error"])
       end
+    end
 
+    describe "when iterate_on is not the same as field, AKA array of objects" do
+      let(:iterate_on_field) { "bar" }
       let(:event) { LogStash::Event.new("foo" => [{"bar"=>"two"},{"bar"=>"one"}, {"bar"=>"six"}]) }
       it "adds a translation to each map" do
         subject.register
@@ -295,10 +324,23 @@ describe LogStash::Filters::Translate do
         expect(event.get("[foo][2][baz]")).to eq("val-6-1|val-6-2")
       end
     end
+
+    describe "when iterate_on is not the same as field, AKA array of objects, coerces integer values to strings" do
+      let(:iterate_on_field) { "bar" }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
+      let(:event) { LogStash::Event.new("foo" => [{"bar"=>200},{"bar"=>300}, {"bar"=>400}]) }
+      it "adds a translation to each map" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("[foo][0][baz]")).to eq("OK")
+        expect(event.get("[foo][1][baz]")).to eq("Redirect")
+        expect(event.get("[foo][2][baz]")).to eq("Client Error")
+      end
+    end
   end
 
   describe "field and destination are the same (needs override)" do
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
     let(:config) do
       {
         "field"            => "foo",
@@ -319,7 +361,7 @@ describe LogStash::Filters::Translate do
   end
 
   describe "general configuration" do
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
     let(:config) do
       {
         "field"            => "random field",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-translate
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.2.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-28 00:00:00.000000000 Z
+date: 2018-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement