logstash-filter-translate 3.2.2 → 3.3.1

data/spec/filters/translate_spec.rb

@@ -1,18 +1,33 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 require "logstash/filters/translate"
 
+module TranslateUtil
+  def self.build_fixture_path(filename)
+    File.join(File.dirname(__FILE__), "..", "fixtures", filename)
+  end
+end
+
 describe LogStash::Filters::Translate do
 
   let(:config) { Hash.new }
   subject { described_class.new(config) }
 
+  let(:logger) { double('Logger').as_null_object }
+  let(:deprecation_logger) { double('DeprecationLogger').as_null_object }
+
+  before(:each) do
+    allow_any_instance_of(described_class).to receive(:logger).and_return(logger)
+    allow_any_instance_of(described_class).to receive(:deprecation_logger).and_return(deprecation_logger)
+  end
+
   describe "exact translation" do
 
     let(:config) do
       {
-        "field"       => "status",
-        "destination" => "translation",
+        "source"      => "status",
+        "target"      => "translation",
         "dictionary"  => [ "200", "OK",
                            "300", "Redirect",
                            "400", "Client Error",
@@ -24,20 +39,43 @@ describe LogStash::Filters::Translate do
 
     let(:event) { LogStash::Event.new("status" => 200) }
 
-    it "return the exact translation" do
+    it "coerces field to a string then returns the exact translation" do
       subject.register
       subject.filter(event)
       expect(event.get("translation")).to eq("OK")
     end
   end
 
+  describe "translation fails when regex setting is false but keys are regex based" do
+
+    let(:config) do
+      {
+        "source"      => "status",
+        "target"      => "translation",
+        "dictionary"  => [ "^2\\d\\d", "OK",
+                           "^3\\d\\d", "Redirect",
+                           "^4\\d\\d", "Client Error",
+                           "^5\\d\\d", "Server Error" ],
+        "exact"       => true,
+        "regex"       => false
+      }
+    end
+
+    let(:event) { LogStash::Event.new("status" => 200) }
+
+    it "does not return the exact translation" do
+      subject.register
+      subject.filter(event)
+      expect(event.get("translation")).to be_nil
+    end
+  end
 
   describe "multi translation" do
     context "when using an inline dictionary" do
       let(:config) do
         {
-          "field"       => "status",
-          "destination" => "translation",
+          "source"      => "status",
+          "target"      => "translation",
           "dictionary"  => [ "200", "OK",
                              "300", "Redirect",
                              "400", "Client Error",
@@ -57,11 +95,11 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a file based dictionary" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "regex_union_dict.csv") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
       let(:config) do
         {
-          "field"       => "status",
-          "destination" => "translation",
+          "source"      => "status",
+          "target"      => "translation",
           "dictionary_path" => dictionary_path,
           "refresh_interval" => 0,
           "exact"       => false,
@@ -83,8 +121,8 @@ describe LogStash::Filters::Translate do
     context "when using an inline dictionary" do
       let(:config) do
         {
-          "field"       => "status",
-          "destination" => "translation",
+          "source"      => "status",
+          "target"      => "translation",
           "dictionary"  => [ "^2[0-9][0-9]$", "OK",
                              "^3[0-9][0-9]$", "Redirect",
                              "^4[0-9][0-9]$", "Client Error",
@@ -104,11 +142,11 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a file based dictionary" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "regex_dict.csv") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_dict.csv") }
       let(:config) do
         {
-          "field"       => "status",
-          "destination" => "translation",
+          "source"      => "status",
+          "target"      => "translation",
           "dictionary_path" => dictionary_path,
           "refresh_interval" => 0,
           "exact"       => true,
@@ -126,53 +164,59 @@ describe LogStash::Filters::Translate do
     end
   end
 
-  describe "fallback value" do
-
-    context "static configuration" do
-      let(:config) do
-        {
-          "field"       => "status",
-          "destination" => "translation",
-          "fallback" => "no match"
-        }
+  describe "fallback value", :ecs_compatibility_support do
+    ecs_compatibility_matrix(:disabled, :v1) do
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
       end
 
-      let(:event) { LogStash::Event.new("status" => "200") }
-
-      it "return the exact translation" do
-        subject.register
-        subject.filter(event)
-        expect(event.get("translation")).to eq("no match")
+      context "static configuration" do
+        let(:config) do
+          {
+              "source"   => "status",
+              "target"   => "translation",
+              "fallback" => "no match"
+          }
+        end
+
+        let(:event) { LogStash::Event.new("status" => "200") }
+
+        it "return the exact translation" do
+          subject.register
+          subject.filter(event)
+          expect(event.get("translation")).to eq("no match")
+        end
       end
-    end
 
-    context "allow sprintf" do
-      let(:config) do
-        {
-          "field"       => "status",
-          "destination" => "translation",
-          "fallback" => "%{missing_translation}"
-        }
+      context "allow sprintf" do
+        let(:config) do
+          {
+              "source"   => "status",
+              "target"   => "translation",
+              "fallback" => "%{missing_translation}"
+          }
+        end
+
+        let(:event) { LogStash::Event.new("status" => "200", "missing_translation" => "missing no match") }
+
+        it "return the exact translation" do
+          subject.register
+          subject.filter(event)
+          expect(event.get("translation")).to eq("missing no match")
+        end
       end
 
-      let(:event) { LogStash::Event.new("status" => "200", "missing_translation" => "missing no match") }
-
-      it "return the exact translation" do
-        subject.register
-        subject.filter(event)
-        expect(event.get("translation")).to eq("missing no match")
-      end
     end
   end
 
   describe "loading a dictionary" do
 
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict-wrong.yml") }
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict-wrong.yml") }
 
     let(:config) do
       {
-        "field"       => "status",
-        "destination" => "translation",
+        "source"      => "status",
+        "target"      => "translation",
         "dictionary_path"  => dictionary_path,
         "refresh_interval" => -1,
         "exact"       => true,
@@ -186,7 +230,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a yml file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
       let(:event) { LogStash::Event.new("status" => "a") }
 
       it "return the exact translation" do
@@ -197,7 +241,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a map tagged yml file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
       let(:event) { LogStash::Event.new("status" => "six") }
 
       it "return the exact translation" do
@@ -208,7 +252,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a omap tagged yml file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-omap-dict.yml") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-omap-dict.yml") }
       let(:event) { LogStash::Event.new("status" => "nine") }
 
       it "return the exact translation" do
@@ -219,7 +263,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a json file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.json") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.json") }
       let(:event) { LogStash::Event.new("status" => "b") }
 
       it "return the exact translation" do
@@ -230,7 +274,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a csv file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.csv") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.csv") }
       let(:event) { LogStash::Event.new("status" => "c") }
 
       it "return the exact translation" do
@@ -241,7 +285,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using an unknown file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.other") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.other") }
 
       it "raises error" do
         expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} has a non valid format/)
@@ -250,42 +294,42 @@ describe LogStash::Filters::Translate do
   end
 
   describe "iterate_on functionality" do
-    describe "when iterate_on is the same as field, AKA array of values" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
-      let(:config) do
-        {
-          "iterate_on"          => "foo",
-          "field"            => "foo",
-          "destination"      => "baz",
-          "fallback"         => "nooo",
-          "dictionary_path"  => dictionary_path,
-          # "override"         => true,
-          "refresh_interval" => 0
-        }
-      end
+    let(:config) do
+      {
+        "iterate_on"       => "foo",
+        "source"           => iterate_on_field,
+        "target"           => "baz",
+        "fallback"         => "nooo",
+        "dictionary_path"  => dictionary_path,
+        # "override"         => true,
+        "refresh_interval" => 0
+      }
+    end
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
 
+    describe "when iterate_on is the same as field, AKA array of values" do
+      let(:iterate_on_field) { "foo" }
       let(:event) { LogStash::Event.new("foo" => ["nine","eight", "seven"]) }
-      it "adds a translation to destination array for each value in field array" do
+      it "adds a translation to target array for each value in field array" do
         subject.register
         subject.filter(event)
         expect(event.get("baz")).to eq(["val-9-1|val-9-2", "val-8-1|val-8-2", "val-7-1|val-7-2"])
       end
     end
 
-    describe "when iterate_on is not the same as field, AKA array of objects" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
-      let(:config) do
-        {
-          "iterate_on"          => "foo",
-          "field"            => "bar",
-          "destination"      => "baz",
-          "fallback"         => "nooo",
-          "dictionary_path"  => dictionary_path,
-          # "override"         => true,
-          "refresh_interval" => 0
-        }
+    describe "when iterate_on is the same as field, AKA array of values, coerces integer elements to strings" do
+      let(:iterate_on_field) { "foo" }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
+      let(:event) { LogStash::Event.new("foo" => [200, 300, 400]) }
+      it "adds a translation to target array for each value in field array" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("baz")).to eq(["OK","Redirect","Client Error"])
       end
+    end
 
+    describe "when iterate_on is not the same as field, AKA array of objects" do
+      let(:iterate_on_field) { "bar" }
       let(:event) { LogStash::Event.new("foo" => [{"bar"=>"two"},{"bar"=>"one"}, {"bar"=>"six"}]) }
       it "adds a translation to each map" do
         subject.register
@@ -295,17 +339,31 @@ describe LogStash::Filters::Translate do
         expect(event.get("[foo][2][baz]")).to eq("val-6-1|val-6-2")
       end
     end
+
+    describe "when iterate_on is not the same as field, AKA array of objects, coerces integer values to strings" do
+      let(:iterate_on_field) { "bar" }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
+      let(:event) { LogStash::Event.new("foo" => [{"bar"=>200},{"bar"=>300}, {"bar"=>400}]) }
+      it "adds a translation to each map" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("[foo][0][baz]")).to eq("OK")
+        expect(event.get("[foo][1][baz]")).to eq("Redirect")
+        expect(event.get("[foo][2][baz]")).to eq("Client Error")
+      end
+    end
   end
 
-  describe "field and destination are the same (needs override)" do
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+  describe "field and destination are the same (explicit override)" do
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
     let(:config) do
       {
         "field"            => "foo",
         "destination"      => "foo",
         "dictionary_path"  => dictionary_path,
         "override"         => true,
-        "refresh_interval" => -1
+        "refresh_interval" => -1,
+        "ecs_compatibility" => 'disabled'
       }
     end
 
@@ -318,11 +376,11 @@ describe LogStash::Filters::Translate do
     end
   end
 
-  describe "general configuration" do
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
+  context "invalid dictionary configuration" do
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
     let(:config) do
       {
-        "field"            => "random field",
+        "source"           => "random field",
         "dictionary"       => { "a" => "b" },
         "dictionary_path"  => dictionary_path,
       }
@@ -333,6 +391,77 @@ describe LogStash::Filters::Translate do
     end
   end
 
+  context "invalid target+destination configuration" do
+    let(:config) do
+      {
+          "source"      => "message",
+          "target"      => 'foo',
+          "destination" => 'bar',
+      }
+    end
+
+    it "raises an exception if both 'target' and 'destination' are set" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError, /remove .*?destination => /)
+    end
+  end
+
+  context "invalid source+field configuration" do
+    let(:config) do
+      {
+        "source"      => "message",
+        "field"       => 'foo'
+      }
+    end
+
+    it "raises an exception if both 'source' and 'field' are set" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError, /remove .*?field => /)
+    end
+  end
+
+  context "destination option" do
+    let(:config) do
+      {
+          "source" => "message", "destination" => 'bar', "ecs_compatibility" => 'v1'
+      }
+    end
+
+    it "sets the target" do
+      subject.register
+      expect( subject.target ).to eql 'bar'
+
+      expect(logger).to have_received(:debug).with(a_string_including "intercepting `destination`")
+      expect(deprecation_logger).to have_received(:deprecated).with(a_string_including "`destination` option is deprecated; use `target` instead.")
+    end
+  end
+
+  context "field option" do
+    let(:config) do
+      {
+          "field" => "message", "target" => 'bar'
+      }
+    end
+
+    it "sets the source" do
+      subject.register # does not raise
+      expect( subject.source ).to eql 'message'
+
+      expect(logger).to have_received(:debug).with(a_string_including "intercepting `field`")
+      expect(deprecation_logger).to have_received(:deprecated).with(a_string_including "`field` option is deprecated; use `source` instead.")
+    end
+  end
+
+  context "source option" do
+    let(:config) do
+      {
+          "target" => 'bar'
+      }
+    end
+
+    it "is required to be set" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError, /provide .*?source => /)
+    end
+  end
+
   describe "refresh_behaviour" do
     let(:dictionary_content) { "a : 1\nb : 2\nc : 3" }
     let(:modified_content) { "a : 1\nb : 4" }
@@ -340,8 +469,8 @@ describe LogStash::Filters::Translate do
     let(:refresh_behaviour) { "merge" }
     let(:config) do
       {
-        "field" => "status",
-        "destination" => "translation",
+        "source" => "status",
+        "target" => "translation",
         "dictionary_path" => dictionary_path,
         "refresh_interval" => -1, # we're controlling this manually
         "exact" => true,
@@ -407,8 +536,8 @@ describe LogStash::Filters::Translate do
 
     let(:config) do
       {
-        "field"       => "status",
-        "destination" => "translation",
+        "source"      => "status",
+        "target"      => "translation",
         "dictionary_path"  => dictionary_path.to_path,
         "refresh_interval" => -1,
         "fallback" => "no match",
@@ -457,4 +586,72 @@ describe LogStash::Filters::Translate do
       end
     end
   end
+
+  describe "default target" do
+
+    let(:config) do
+      {
+          "source" => "message",
+          "dictionary" => { "foo" => "bar" }
+      }
+    end
+
+    let(:event) { LogStash::Event.new("message" => "foo") }
+
+    before { subject.register }
+
+    context "legacy mode" do
+
+      let(:config) { super().merge('ecs_compatibility' => 'disabled') }
+
+      it "uses the translation target" do
+        subject.filter(event)
+        expect(event.get("translation")).to eq("bar")
+        expect(event.get("message")).to eq("foo")
+      end
+
+    end
+
+    context "ECS mode" do
+
+      let(:config) { super().merge('ecs_compatibility' => 'v1') }
+
+      it "does in place translation" do
+        subject.filter(event)
+        expect(event.include?("translation")).to be false
+        expect(event.get("message")).to eq("bar")
+      end
+
+    end
+
+  end
+
+
+  describe "error handling" do
+
+    let(:config) do
+      {
+          "source" => "message",
+          "dictionary" => { "foo" => "bar" }
+      }
+    end
+
+    let(:event) { LogStash::Event.new("message" => "foo") }
+
+    before { subject.register }
+
+    it "handles unexpected error within filter" do
+      expect(subject.updater).to receive(:update).and_raise RuntimeError.new('TEST')
+
+      expect { subject.filter(event) }.to_not raise_error
+    end
+
+    it "propagates Java errors" do
+      expect(subject.updater).to receive(:update).and_raise java.lang.OutOfMemoryError.new('FAKE-OUT!')
+
+      expect { subject.filter(event) }.to raise_error(java.lang.OutOfMemoryError)
+    end
+
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-translate
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.3.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-28 00:00:00.000000000 Z
+date: 2022-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,48 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-deprecation_logger_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -164,8 +206,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Replaces field contents based on a hash or YAML file