logstash-filter-translate 3.1.0 → 3.3.0

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.1.0'
+  s.version         = '3.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,14 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.2'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~> 1.0'
+  s.add_runtime_dependency 'rufus-scheduler'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'rspec-sequencing'
+  s.add_development_dependency "rspec-wait"
+  s.add_development_dependency "benchmark-ips"
 end
 

data/spec/filters/benchmark_rspec.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+
+require "logstash/filters/translate"
+require "benchmark/ips"
+
+module BenchmarkingFileBuilder
+  def self.create_huge_csv_dictionary(directory, name, size)
+    tmppath = directory.join("temp_big.csv")
+    tmppath.open("w") do |file|
+      file.puts("foo,#{SecureRandom.hex(4)}")
+      file.puts("bar,#{SecureRandom.hex(4)}")
+      size.times do |i|
+        file.puts("#{SecureRandom.hex(12)},#{1000000 + i}")
+      end
+      file.puts("baz,quux")
+    end
+    tmppath.rename(directory.join(name))
+  end
+end
+
+describe LogStash::Filters::Translate do
+  let(:directory) { Pathname.new(Stud::Temporary.directory) }
+  let(:dictionary_name) { "dict-h.csv" }
+  let(:dictionary_path) { directory.join(dictionary_name) }
+  let(:dictionary_size) { 100000 }
+  let(:config) do
+    {
+      "field"       => "[status]",
+      "destination" => "[translation]",
+      "dictionary_path"  => dictionary_path.to_path,
+      "exact"       => true,
+      "regex"       => false,
+      "refresh_interval" => 0,
+      "override" => true,
+      "refresh_behaviour" => "merge"
+    }
+  end
+  before do
+    directory
+    BenchmarkingFileBuilder.create_huge_csv_dictionary(directory, dictionary_name, dictionary_size)
+  end
+
+  it 'how fast is the filter method?' do
+    plugin = described_class.new(config)
+    plugin.register
+    event = LogStash::Event.new("status" => "baz", "translation" => "foo")
+
+    Benchmark.ips do |x|
+      x.config(:time => 20, :warmup => 120)
+      x.report("filter(event)") { plugin.filter(event) }
+    end
+    expect(event.get("[translation]")).to eq("quux")
+  end
+
+  it 'how fast is the new, register then filter method?' do
+    event = LogStash::Event.new("status" => "baz", "translation" => "foo")
+
+    Benchmark.ips do |x|
+      x.config(:time => 10, :warmup => 120)
+      x.report("new, register, filter(event)") do
+        plugin = described_class.new(config)
+        plugin.register
+        plugin.filter(event)
+       end
+    end
+    expect(event.get("[translation]")).to eq("quux")
+  end
+end

data/spec/filters/scheduling_spec.rb

@@ -0,0 +1,201 @@
+# encoding: utf-8
+require 'rspec/wait'
+require "logstash/devutils/rspec/spec_helper"
+require "support/rspec_wait_handler_helper"
+require "support/build_huge_dictionaries"
+
+require "rspec_sequencing"
+
+require "logstash/filters/translate"
+
+describe LogStash::Filters::Translate do
+  let(:directory) { Pathname.new(Stud::Temporary.directory) }
+  describe "scheduled reloading" do
+    subject { described_class.new(config) }
+
+    let(:config) do
+      {
+        "source"      => "[status]",
+        "target"      => "[translation]",
+        "dictionary_path"  => dictionary_path.to_path,
+        "exact"       => true,
+        "regex"       => false,
+        "refresh_interval" => 1,
+        "override" => true,
+        "refresh_behaviour" => refresh_behaviour
+      }
+    end
+
+    let(:event) { LogStash::Event.new("status" => "b") }
+
+    before do
+      directory
+      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      dictionary_path.open("wb") do |file|
+        file.puts("a,1\nb,2\nc,3\n")
+      end
+      subject.register
+    end
+
+    after do
+      FileUtils.rm_rf(directory)
+      wait(1.0).for{Dir.exist?(directory)}.to eq(false)
+    end
+
+    context "replace" do
+      let(:dictionary_path) { directory.join("dict-r.csv") }
+      let(:refresh_behaviour) { "replace" }
+      let(:actions) do
+        RSpec::Sequencing
+          .run("translate") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("2"), "field [translation] did not eq '2'"
+          end
+          .then_after(1,"modify file") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,11\nb,12\nc,13\n")
+            end
+          end
+          .then_after(1.2, "wait then translate again") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("12"), "field [translation] did not eq '12'"
+          end
+          .then("stop") do
+            subject.close
+          end
+      end
+
+      it "updates the event after scheduled reload" do
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+    end
+
+    context "merge" do
+      let(:dictionary_path) { directory.join("dict-m.csv") }
+      let(:refresh_behaviour) { "merge" }
+      let(:actions) do
+        RSpec::Sequencing
+          .run("translate") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("2"), "field [translation] did not eq '2'"
+          end
+          .then_after(1,"modify file") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,21\nb,22\nc,23\n")
+            end
+          end
+          .then_after(1.2, "wait then translate again") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("22"), "field [translation] did not eq '22'"
+          end
+          .then("stop") do
+            subject.close
+          end
+      end
+
+      it "updates the event after scheduled reload" do
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+    end
+  end
+
+  describe "huge json file merge" do
+    let(:dictionary_path) { directory.join("dict-h.json") }
+    let(:dictionary_size) { 100000 }
+    let(:config) do
+      {
+        "source"      => "[status]",
+        "target"      => "[translation]",
+        "dictionary_path"  => dictionary_path.to_path,
+        "exact"       => true,
+        "regex"       => false,
+        "refresh_interval" => 1,
+        "override" => true,
+        "refresh_behaviour" => "merge"
+      }
+    end
+    let(:event) { LogStash::Event.new("status" => "baz", "translation" => "foo") }
+    subject { described_class.new(config) }
+
+    before do
+      directory
+      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      LogStash::Filters::Dictionary.create_huge_json_dictionary(directory, "dict-h.json", dictionary_size)
+      subject.register
+    end
+
+    let(:actions) do
+      RSpec::Sequencing
+        .run("translate") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then_after(0.1,"modify file") do
+          LogStash::Filters::Dictionary.create_huge_json_dictionary(directory, "dict-h.json", dictionary_size)
+        end
+        .then_after(1.8, "wait then translate again") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then("stop") do
+          subject.close
+        end
+    end
+
+    it "updates the event after scheduled reload" do
+      actions.activate_quietly
+      actions.assert_no_errors
+    end
+  end
+
+  describe "huge csv file merge" do
+    let(:dictionary_path) { directory.join("dict-h.csv") }
+    let(:dictionary_size) { 100000 }
+    let(:config) do
+      {
+        "source"      => "[status]",
+        "target"      => "[translation]",
+        "dictionary_path"  => dictionary_path.to_path,
+        "exact"       => true,
+        "regex"       => false,
+        "refresh_interval" => 1,
+        "override" => true,
+        "refresh_behaviour" => "merge"
+      }
+    end
+    let(:event) { LogStash::Event.new("status" => "bar", "translation" => "foo") }
+    subject { described_class.new(config) }
+
+    before do
+      directory
+      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      LogStash::Filters::Dictionary.create_huge_csv_dictionary(directory, "dict-h.csv", dictionary_size)
+      subject.register
+    end
+
+    let(:actions) do
+      RSpec::Sequencing
+        .run("translate") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then_after(0.1,"modify file") do
+          LogStash::Filters::Dictionary.create_huge_csv_dictionary(directory, "dict-h.csv", dictionary_size)
+        end
+        .then_after(1.8, "wait then translate again") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then("stop") do
+          subject.close
+        end
+    end
+
+    it "updates the event after scheduled reload" do
+      actions.activate_quietly
+      actions.assert_no_errors
+    end
+  end
+end

data/spec/filters/translate_spec.rb

@@ -1,146 +1,236 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 require "logstash/filters/translate"
 
+module TranslateUtil
+  def self.build_fixture_path(filename)
+    File.join(File.dirname(__FILE__), "..", "fixtures", filename)
+  end
+end
+
 describe LogStash::Filters::Translate do
 
   let(:config) { Hash.new }
   subject { described_class.new(config) }
 
+  let(:logger) { double('Logger').as_null_object }
+  let(:deprecation_logger) { double('DeprecationLogger').as_null_object }
+
+  before(:each) do
+    allow_any_instance_of(described_class).to receive(:logger).and_return(logger)
+    allow_any_instance_of(described_class).to receive(:deprecation_logger).and_return(deprecation_logger)
+  end
+
   describe "exact translation" do
 
     let(:config) do
       {
-        "field"       => "status",
-        "destination" => "translation",
+        "source"      => "status",
+        "target"      => "translation",
         "dictionary"  => [ "200", "OK",
                            "300", "Redirect",
                            "400", "Client Error",
                            "500", "Server Error" ],
-                           "exact"       => true,
-                           "regex"       => false
+        "exact"       => true,
+        "regex"       => false
       }
     end
 
     let(:event) { LogStash::Event.new("status" => 200) }
 
-    it "return the exact translation" do
+    it "coerces field to a string then returns the exact translation" do
       subject.register
       subject.filter(event)
       expect(event.get("translation")).to eq("OK")
     end
   end
 
-
-  describe "multi translation" do
+  describe "translation fails when regex setting is false but keys are regex based" do
 
     let(:config) do
       {
-        "field"       => "status",
-        "destination" => "translation",
-        "dictionary"  => [ "200", "OK",
-                           "300", "Redirect",
-                           "400", "Client Error",
-                           "500", "Server Error" ],
-                           "exact"       => false,
-                           "regex"       => false
+        "source"      => "status",
+        "target"      => "translation",
+        "dictionary"  => [ "^2\\d\\d", "OK",
+                           "^3\\d\\d", "Redirect",
+                           "^4\\d\\d", "Client Error",
+                           "^5\\d\\d", "Server Error" ],
+        "exact"       => true,
+        "regex"       => false
       }
     end
 
-    let(:event) { LogStash::Event.new("status" => "200 & 500") }
+    let(:event) { LogStash::Event.new("status" => 200) }
 
-    it "return the exact translation" do
+    it "does not return the exact translation" do
       subject.register
       subject.filter(event)
-      expect(event.get("translation")).to eq("OK & Server Error")
+      expect(event.get("translation")).to be_nil
     end
-
   end
 
-  describe "regex translation" do
+  describe "multi translation" do
+    context "when using an inline dictionary" do
+      let(:config) do
+        {
+          "source"      => "status",
+          "target"      => "translation",
+          "dictionary"  => [ "200", "OK",
+                             "300", "Redirect",
+                             "400", "Client Error",
+                             "500", "Server Error" ],
+          "exact"       => false,
+          "regex"       => false
+        }
+      end
 
-    let(:config) do
-      {
-        "field"       => "status",
-        "destination" => "translation",
-        "dictionary"  => [ "^2[0-9][0-9]$", "OK",
-                           "^3[0-9][0-9]$", "Redirect",
-                           "^4[0-9][0-9]$", "Client Error",
-                           "^5[0-9][0-9]$", "Server Error" ],
-        "exact"       => true,
-        "regex"       => true
-      }
+      let(:event) { LogStash::Event.new("status" => "200 & 500") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("OK & Server Error")
+      end
     end
 
-    let(:event) { LogStash::Event.new("status" => "200") }
+    context "when using a file based dictionary" do
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
+      let(:config) do
+        {
+          "source"      => "status",
+          "target"      => "translation",
+          "dictionary_path" => dictionary_path,
+          "refresh_interval" => 0,
+          "exact"       => false,
+          "regex"       => false
+        }
+      end
 
-    it "return the exact translation" do
-      subject.register
-      subject.filter(event)
-      expect(event.get("translation")).to eq("OK")
+      let(:event) { LogStash::Event.new("status" => "200 & 500") }
+
+      it "return the exact regex translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("OK & Server Error")
+      end
     end
   end
 
-  describe "fallback value" do
-
-    context "static configuration" do
+  describe "regex translation" do
+    context "when using an inline dictionary" do
       let(:config) do
         {
-          "field"       => "status",
-          "destination" => "translation",
-          "fallback" => "no match"
+          "source"      => "status",
+          "target"      => "translation",
+          "dictionary"  => [ "^2[0-9][0-9]$", "OK",
+                             "^3[0-9][0-9]$", "Redirect",
+                             "^4[0-9][0-9]$", "Client Error",
+                             "^5[0-9][0-9]$", "Server Error" ],
+          "exact"       => true,
+          "regex"       => true
         }
       end
 
       let(:event) { LogStash::Event.new("status" => "200") }
 
-      it "return the exact translation" do
+      it "return the exact regex translation" do
         subject.register
         subject.filter(event)
-        expect(event.get("translation")).to eq("no match")
+        expect(event.get("translation")).to eq("OK")
       end
     end
 
-    context "allow sprintf" do
+    context "when using a file based dictionary" do
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_dict.csv") }
       let(:config) do
         {
-          "field"       => "status",
-          "destination" => "translation",
-          "fallback" => "%{missing_translation}"
+          "source"      => "status",
+          "target"      => "translation",
+          "dictionary_path" => dictionary_path,
+          "refresh_interval" => 0,
+          "exact"       => true,
+          "regex"       => true
         }
       end
 
-      let(:event) { LogStash::Event.new("status" => "200", "missing_translation" => "missing no match") }
+      let(:event) { LogStash::Event.new("status" => "200") }
 
-      it "return the exact translation" do
+      it "return the exact regex translation" do
         subject.register
         subject.filter(event)
-        expect(event.get("translation")).to eq("missing no match")
+        expect(event.get("translation")).to eq("OK")
       end
     end
   end
 
+  describe "fallback value", :ecs_compatibility_support do
+    ecs_compatibility_matrix(:disabled, :v1) do
+      before(:each) do
+        allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+      end
+
+      context "static configuration" do
+        let(:config) do
+          {
+              "source"   => "status",
+              "target"   => "translation",
+              "fallback" => "no match"
+          }
+        end
+
+        let(:event) { LogStash::Event.new("status" => "200") }
+
+        it "return the exact translation" do
+          subject.register
+          subject.filter(event)
+          expect(event.get("translation")).to eq("no match")
+        end
+      end
+
+      context "allow sprintf" do
+        let(:config) do
+          {
+              "source"   => "status",
+              "target"   => "translation",
+              "fallback" => "%{missing_translation}"
+          }
+        end
+
+        let(:event) { LogStash::Event.new("status" => "200", "missing_translation" => "missing no match") }
+
+        it "return the exact translation" do
+          subject.register
+          subject.filter(event)
+          expect(event.get("translation")).to eq("missing no match")
+        end
+      end
+
+    end
+  end
+
   describe "loading a dictionary" do
 
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict-wrong.yml") }
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict-wrong.yml") }
 
     let(:config) do
       {
-        "field"       => "status",
-        "destination" => "translation",
+        "source"      => "status",
+        "target"      => "translation",
         "dictionary_path"  => dictionary_path,
+        "refresh_interval" => -1,
         "exact"       => true,
         "regex"       => false
       }
     end
 
     it "raises exception when loading" do
-      error = "(#{dictionary_path}): mapping values are not allowed here at line 1 column 45 when loading dictionary file at #{dictionary_path}"
-      expect { subject.register }.to raise_error("#{described_class}: #{error}")
+      error = /mapping values are not allowed here at line 1 column 45 when loading dictionary file/
+      expect { subject.register }.to raise_error(error)
     end
 
     context "when using a yml file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
       let(:event) { LogStash::Event.new("status" => "a") }
 
       it "return the exact translation" do
@@ -150,8 +240,30 @@ describe LogStash::Filters::Translate do
       end
     end
 
+    context "when using a map tagged yml file" do
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
+      let(:event) { LogStash::Event.new("status" => "six") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("val-6-1|val-6-2")
+      end
+    end
+
+    context "when using a omap tagged yml file" do
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-omap-dict.yml") }
+      let(:event) { LogStash::Event.new("status" => "nine") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("val-9-1|val-9-2")
+      end
+    end
+
     context "when using a json file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.json") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.json") }
       let(:event) { LogStash::Event.new("status" => "b") }
 
       it "return the exact translation" do
@@ -162,7 +274,7 @@ describe LogStash::Filters::Translate do
     end
 
     context "when using a csv file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.csv") }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.csv") }
       let(:event) { LogStash::Event.new("status" => "c") }
 
       it "return the exact translation" do
@@ -172,20 +284,103 @@ describe LogStash::Filters::Translate do
       end
     end
 
-    context "when using an uknown file" do
-      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.other") }
+    context "when using an unknown file" do
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.other") }
 
-      it "return the exact translation" do
-        expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} have a non valid format/)
+      it "raises error" do
+        expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} has a non valid format/)
+      end
+    end
+  end
+
+  describe "iterate_on functionality" do
+    let(:config) do
+      {
+        "iterate_on"       => "foo",
+        "source"           => iterate_on_field,
+        "target"           => "baz",
+        "fallback"         => "nooo",
+        "dictionary_path"  => dictionary_path,
+        # "override"         => true,
+        "refresh_interval" => 0
+      }
+    end
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
+
+    describe "when iterate_on is the same as field, AKA array of values" do
+      let(:iterate_on_field) { "foo" }
+      let(:event) { LogStash::Event.new("foo" => ["nine","eight", "seven"]) }
+      it "adds a translation to target array for each value in field array" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("baz")).to eq(["val-9-1|val-9-2", "val-8-1|val-8-2", "val-7-1|val-7-2"])
+      end
+    end
+
+    describe "when iterate_on is the same as field, AKA array of values, coerces integer elements to strings" do
+      let(:iterate_on_field) { "foo" }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
+      let(:event) { LogStash::Event.new("foo" => [200, 300, 400]) }
+      it "adds a translation to target array for each value in field array" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("baz")).to eq(["OK","Redirect","Client Error"])
       end
     end
+
+    describe "when iterate_on is not the same as field, AKA array of objects" do
+      let(:iterate_on_field) { "bar" }
+      let(:event) { LogStash::Event.new("foo" => [{"bar"=>"two"},{"bar"=>"one"}, {"bar"=>"six"}]) }
+      it "adds a translation to each map" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("[foo][0][baz]")).to eq("val-2-1|val-2-2")
+        expect(event.get("[foo][1][baz]")).to eq("val-1-1|val-1-2")
+        expect(event.get("[foo][2][baz]")).to eq("val-6-1|val-6-2")
+      end
+    end
+
+    describe "when iterate_on is not the same as field, AKA array of objects, coerces integer values to strings" do
+      let(:iterate_on_field) { "bar" }
+      let(:dictionary_path)  { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
+      let(:event) { LogStash::Event.new("foo" => [{"bar"=>200},{"bar"=>300}, {"bar"=>400}]) }
+      it "adds a translation to each map" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("[foo][0][baz]")).to eq("OK")
+        expect(event.get("[foo][1][baz]")).to eq("Redirect")
+        expect(event.get("[foo][2][baz]")).to eq("Client Error")
+      end
+    end
+  end
+
+  describe "field and destination are the same (explicit override)" do
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
+    let(:config) do
+      {
+        "field"            => "foo",
+        "destination"      => "foo",
+        "dictionary_path"  => dictionary_path,
+        "override"         => true,
+        "refresh_interval" => -1,
+        "ecs_compatibility" => 'disabled'
+      }
+    end
+
+    let(:event) { LogStash::Event.new("foo" => "nine") }
+
+    it "overwrites existing value" do
+      subject.register
+      subject.filter(event)
+      expect(event.get("foo")).to eq("val-9-1|val-9-2")
+    end
   end
 
-  describe "general configuration" do
-    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
+  context "invalid dictionary configuration" do
+    let(:dictionary_path)  { TranslateUtil.build_fixture_path("dict.yml") }
     let(:config) do
       {
-        "field"            => "random field",
+        "source"           => "random field",
         "dictionary"       => { "a" => "b" },
         "dictionary_path"  => dictionary_path,
       }
@@ -196,6 +391,77 @@ describe LogStash::Filters::Translate do
     end
   end
 
+  context "invalid target+destination configuration" do
+    let(:config) do
+      {
+          "source"      => "message",
+          "target"      => 'foo',
+          "destination" => 'bar',
+      }
+    end
+
+    it "raises an exception if both 'target' and 'destination' are set" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError, /remove .*?destination => /)
+    end
+  end
+
+  context "invalid source+field configuration" do
+    let(:config) do
+      {
+        "source"      => "message",
+        "field"       => 'foo'
+      }
+    end
+
+    it "raises an exception if both 'source' and 'field' are set" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError, /remove .*?field => /)
+    end
+  end
+
+  context "destination option" do
+    let(:config) do
+      {
+          "source" => "message", "destination" => 'bar', "ecs_compatibility" => 'v1'
+      }
+    end
+
+    it "sets the target" do
+      subject.register
+      expect( subject.target ).to eql 'bar'
+
+      expect(logger).to have_received(:debug).with(a_string_including "intercepting `destination`")
+      expect(deprecation_logger).to have_received(:deprecated).with(a_string_including "`destination` option is deprecated; use `target` instead.")
+    end
+  end
+
+  context "field option" do
+    let(:config) do
+      {
+          "field" => "message", "target" => 'bar'
+      }
+    end
+
+    it "sets the source" do
+      subject.register # does not raise
+      expect( subject.source ).to eql 'message'
+
+      expect(logger).to have_received(:debug).with(a_string_including "intercepting `field`")
+      expect(deprecation_logger).to have_received(:deprecated).with(a_string_including "`field` option is deprecated; use `source` instead.")
+    end
+  end
+
+  context "source option" do
+    let(:config) do
+      {
+          "target" => 'bar'
+      }
+    end
+
+    it "is required to be set" do
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError, /provide .*?source => /)
+    end
+  end
+
   describe "refresh_behaviour" do
     let(:dictionary_content) { "a : 1\nb : 2\nc : 3" }
     let(:modified_content) { "a : 1\nb : 4" }
@@ -203,10 +469,10 @@ describe LogStash::Filters::Translate do
     let(:refresh_behaviour) { "merge" }
     let(:config) do
       {
-        "field" => "status",
-        "destination" => "translation",
+        "source" => "status",
+        "target" => "translation",
         "dictionary_path" => dictionary_path,
-        "refresh_interval" => 10000, # we're controlling this manually
+        "refresh_interval" => -1, # we're controlling this manually
         "exact" => true,
         "regex" => false,
         "fallback" => "no match",
@@ -229,7 +495,7 @@ describe LogStash::Filters::Translate do
       it "overwrites existing entries" do
         subject.filter(before_mod)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_mod)
         expect(before_mod.get("translation")).to eq(2)
         expect(after_mod.get("translation")).to eq(4)
@@ -237,7 +503,7 @@ describe LogStash::Filters::Translate do
       it "keeps leftover entries" do
         subject.filter(before_del)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_del)
         expect(before_del.get("translation")).to eq(3)
         expect(after_del.get("translation")).to eq(3)
@@ -249,7 +515,7 @@ describe LogStash::Filters::Translate do
       it "overwrites existing entries" do
         subject.filter(before_mod)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_mod)
         expect(before_mod.get("translation")).to eq(2)
         expect(after_mod.get("translation")).to eq(4)
@@ -257,11 +523,135 @@ describe LogStash::Filters::Translate do
       it "removes leftover entries" do
         subject.filter(before_del)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_del)
         expect(before_del.get("translation")).to eq(3)
         expect(after_del.get("translation")).to eq("no match")
       end
     end
   end
+
+  describe "loading an empty dictionary" do
+    let(:directory) { Pathname.new(Stud::Temporary.directory) }
+
+    let(:config) do
+      {
+        "source"      => "status",
+        "target"      => "translation",
+        "dictionary_path"  => dictionary_path.to_path,
+        "refresh_interval" => -1,
+        "fallback" => "no match",
+        "exact"       => true,
+        "regex"       => false
+      }
+    end
+
+    before do
+      dictionary_path.open("wb") do |file|
+        file.write("")
+      end
+    end
+
+    context "when using a yml file" do
+      let(:dictionary_path) { directory.join("dict-e.yml") }
+      let(:event) { LogStash::Event.new("status" => "a") }
+
+      it "return the exact translation" do
+
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("no match")
+      end
+    end
+
+    context "when using a json file" do
+      let(:dictionary_path) { directory.join("dict-e.json") }
+      let(:event) { LogStash::Event.new("status" => "b") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("no match")
+      end
+    end
+
+    context "when using a csv file" do
+      let(:dictionary_path) { directory.join("dict-e.csv") }
+      let(:event) { LogStash::Event.new("status" => "c") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("no match")
+      end
+    end
+  end
+
+  describe "default target" do
+
+    let(:config) do
+      {
+          "source" => "message",
+          "dictionary" => { "foo" => "bar" }
+      }
+    end
+
+    let(:event) { LogStash::Event.new("message" => "foo") }
+
+    before { subject.register }
+
+    context "legacy mode" do
+
+      let(:config) { super().merge('ecs_compatibility' => 'disabled') }
+
+      it "uses the translation target" do
+        subject.filter(event)
+        expect(event.get("translation")).to eq("bar")
+        expect(event.get("message")).to eq("foo")
+      end
+
+    end
+
+    context "ECS mode" do
+
+      let(:config) { super().merge('ecs_compatibility' => 'v1') }
+
+      it "does in place translation" do
+        subject.filter(event)
+        expect(event.include?("translation")).to be false
+        expect(event.get("message")).to eq("bar")
+      end
+
+    end
+
+  end
+
+
+  describe "error handling" do
+
+    let(:config) do
+      {
+          "source" => "message",
+          "dictionary" => { "foo" => "bar" }
+      }
+    end
+
+    let(:event) { LogStash::Event.new("message" => "foo") }
+
+    before { subject.register }
+
+    it "handles unexpected error within filter" do
+      expect(subject.updater).to receive(:update).and_raise RuntimeError.new('TEST')
+
+      expect { subject.filter(event) }.to_not raise_error
+    end
+
+    it "propagates Java errors" do
+      expect(subject.updater).to receive(:update).and_raise java.lang.OutOfMemoryError.new('FAKE-OUT!')
+
+      expect { subject.filter(event) }.to raise_error(java.lang.OutOfMemoryError)
+    end
+
+  end
+
 end