logstash-filter-translate 3.1.0 → 3.2.0

data/spec/filters/translate_spec.rb

@@ -17,8 +17,8 @@ describe LogStash::Filters::Translate do
                            "300", "Redirect",
                            "400", "Client Error",
                            "500", "Server Error" ],
-                           "exact"       => true,
-                           "regex"       => false
+        "exact"       => true,
+        "regex"       => false
       }
     end
 
@@ -33,51 +33,96 @@ describe LogStash::Filters::Translate do
 
 
   describe "multi translation" do
+    context "when using an inline dictionary" do
+      let(:config) do
+        {
+          "field"       => "status",
+          "destination" => "translation",
+          "dictionary"  => [ "200", "OK",
+                             "300", "Redirect",
+                             "400", "Client Error",
+                             "500", "Server Error" ],
+          "exact"       => false,
+          "regex"       => false
+        }
+      end
 
-    let(:config) do
-      {
-        "field"       => "status",
-        "destination" => "translation",
-        "dictionary"  => [ "200", "OK",
-                           "300", "Redirect",
-                           "400", "Client Error",
-                           "500", "Server Error" ],
-                           "exact"       => false,
-                           "regex"       => false
-      }
+      let(:event) { LogStash::Event.new("status" => "200 & 500") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("OK & Server Error")
+      end
     end
 
-    let(:event) { LogStash::Event.new("status" => "200 & 500") }
+    context "when using a file based dictionary" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "regex_union_dict.csv") }
+      let(:config) do
+        {
+          "field"       => "status",
+          "destination" => "translation",
+          "dictionary_path" => dictionary_path,
+          "refresh_interval" => 0,
+          "exact"       => false,
+          "regex"       => false
+        }
+      end
 
-    it "return the exact translation" do
-      subject.register
-      subject.filter(event)
-      expect(event.get("translation")).to eq("OK & Server Error")
-    end
+      let(:event) { LogStash::Event.new("status" => "200 & 500") }
 
+      it "return the exact regex translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("OK & Server Error")
+      end
+    end
   end
 
   describe "regex translation" do
+    context "when using an inline dictionary" do
+      let(:config) do
+        {
+          "field"       => "status",
+          "destination" => "translation",
+          "dictionary"  => [ "^2[0-9][0-9]$", "OK",
+                             "^3[0-9][0-9]$", "Redirect",
+                             "^4[0-9][0-9]$", "Client Error",
+                             "^5[0-9][0-9]$", "Server Error" ],
+          "exact"       => true,
+          "regex"       => true
+        }
+      end
 
-    let(:config) do
-      {
-        "field"       => "status",
-        "destination" => "translation",
-        "dictionary"  => [ "^2[0-9][0-9]$", "OK",
-                           "^3[0-9][0-9]$", "Redirect",
-                           "^4[0-9][0-9]$", "Client Error",
-                           "^5[0-9][0-9]$", "Server Error" ],
-        "exact"       => true,
-        "regex"       => true
-      }
+      let(:event) { LogStash::Event.new("status" => "200") }
+
+      it "return the exact regex translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("OK")
+      end
     end
 
-    let(:event) { LogStash::Event.new("status" => "200") }
+    context "when using a file based dictionary" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "regex_dict.csv") }
+      let(:config) do
+        {
+          "field"       => "status",
+          "destination" => "translation",
+          "dictionary_path" => dictionary_path,
+          "refresh_interval" => 0,
+          "exact"       => true,
+          "regex"       => true
+        }
+      end
 
-    it "return the exact translation" do
-      subject.register
-      subject.filter(event)
-      expect(event.get("translation")).to eq("OK")
+      let(:event) { LogStash::Event.new("status" => "200") }
+
+      it "return the exact regex translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("OK")
+      end
     end
   end
 
@@ -129,14 +174,15 @@ describe LogStash::Filters::Translate do
         "field"       => "status",
         "destination" => "translation",
         "dictionary_path"  => dictionary_path,
+        "refresh_interval" => -1,
         "exact"       => true,
         "regex"       => false
       }
     end
 
     it "raises exception when loading" do
-      error = "(#{dictionary_path}): mapping values are not allowed here at line 1 column 45 when loading dictionary file at #{dictionary_path}"
-      expect { subject.register }.to raise_error("#{described_class}: #{error}")
+      error = /mapping values are not allowed here at line 1 column 45 when loading dictionary file/
+      expect { subject.register }.to raise_error(error)
     end
 
     context "when using a yml file" do
@@ -150,6 +196,28 @@ describe LogStash::Filters::Translate do
       end
     end
 
+    context "when using a map tagged yml file" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+      let(:event) { LogStash::Event.new("status" => "six") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("val-6-1|val-6-2")
+      end
+    end
+
+    context "when using a omap tagged yml file" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-omap-dict.yml") }
+      let(:event) { LogStash::Event.new("status" => "nine") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("val-9-1|val-9-2")
+      end
+    end
+
     context "when using a json file" do
       let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.json") }
       let(:event) { LogStash::Event.new("status" => "b") }
@@ -172,15 +240,84 @@ describe LogStash::Filters::Translate do
       end
     end
 
-    context "when using an uknown file" do
+    context "when using an unknown file" do
       let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.other") }
 
-      it "return the exact translation" do
-        expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} have a non valid format/)
+      it "raises error" do
+        expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} has a non valid format/)
       end
     end
   end
 
+  describe "iterate_on functionality" do
+    describe "when iterate_on is the same as field, AKA array of values" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+      let(:config) do
+        {
+          "iterate_on"          => "foo",
+          "field"            => "foo",
+          "destination"      => "baz",
+          "fallback"         => "nooo",
+          "dictionary_path"  => dictionary_path,
+          # "override"         => true,
+          "refresh_interval" => 0
+        }
+      end
+
+      let(:event) { LogStash::Event.new("foo" => ["nine","eight", "seven"]) }
+      it "adds a translation to destination array for each value in field array" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("baz")).to eq(["val-9-1|val-9-2", "val-8-1|val-8-2", "val-7-1|val-7-2"])
+      end
+    end
+
+    describe "when iterate_on is not the same as field, AKA array of objects" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+      let(:config) do
+        {
+          "iterate_on"          => "foo",
+          "field"            => "bar",
+          "destination"      => "baz",
+          "fallback"         => "nooo",
+          "dictionary_path"  => dictionary_path,
+          # "override"         => true,
+          "refresh_interval" => 0
+        }
+      end
+
+      let(:event) { LogStash::Event.new("foo" => [{"bar"=>"two"},{"bar"=>"one"}, {"bar"=>"six"}]) }
+      it "adds a translation to each map" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("[foo][0][baz]")).to eq("val-2-1|val-2-2")
+        expect(event.get("[foo][1][baz]")).to eq("val-1-1|val-1-2")
+        expect(event.get("[foo][2][baz]")).to eq("val-6-1|val-6-2")
+      end
+    end
+  end
+
+  describe "field and destination are the same (needs override)" do
+    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "tag-map-dict.yml") }
+    let(:config) do
+      {
+        "field"            => "foo",
+        "destination"      => "foo",
+        "dictionary_path"  => dictionary_path,
+        "override"         => true,
+        "refresh_interval" => -1
+      }
+    end
+
+    let(:event) { LogStash::Event.new("foo" => "nine") }
+
+    it "overwrites existing value" do
+      subject.register
+      subject.filter(event)
+      expect(event.get("foo")).to eq("val-9-1|val-9-2")
+    end
+  end
+
   describe "general configuration" do
     let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
     let(:config) do
@@ -206,7 +343,7 @@ describe LogStash::Filters::Translate do
         "field" => "status",
         "destination" => "translation",
         "dictionary_path" => dictionary_path,
-        "refresh_interval" => 10000, # we're controlling this manually
+        "refresh_interval" => -1, # we're controlling this manually
         "exact" => true,
         "regex" => false,
         "fallback" => "no match",
@@ -229,7 +366,7 @@ describe LogStash::Filters::Translate do
       it "overwrites existing entries" do
         subject.filter(before_mod)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_mod)
         expect(before_mod.get("translation")).to eq(2)
         expect(after_mod.get("translation")).to eq(4)
@@ -237,7 +374,7 @@ describe LogStash::Filters::Translate do
       it "keeps leftover entries" do
         subject.filter(before_del)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_del)
         expect(before_del.get("translation")).to eq(3)
         expect(after_del.get("translation")).to eq(3)
@@ -249,7 +386,7 @@ describe LogStash::Filters::Translate do
       it "overwrites existing entries" do
         subject.filter(before_mod)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_mod)
         expect(before_mod.get("translation")).to eq(2)
         expect(after_mod.get("translation")).to eq(4)
@@ -257,11 +394,67 @@ describe LogStash::Filters::Translate do
       it "removes leftover entries" do
         subject.filter(before_del)
         IO.write(dictionary_path, modified_content)
-        subject.send(:load_dictionary)
+        subject.lookup.load_dictionary
         subject.filter(after_del)
         expect(before_del.get("translation")).to eq(3)
         expect(after_del.get("translation")).to eq("no match")
       end
     end
   end
+
+  describe "loading an empty dictionary" do
+    let(:directory) { Pathname.new(Stud::Temporary.directory) }
+
+    let(:config) do
+      {
+        "field"       => "status",
+        "destination" => "translation",
+        "dictionary_path"  => dictionary_path.to_path,
+        "refresh_interval" => -1,
+        "fallback" => "no match",
+        "exact"       => true,
+        "regex"       => false
+      }
+    end
+
+    before do
+      dictionary_path.open("wb") do |file|
+        file.write("")
+      end
+    end
+
+    context "when using a yml file" do
+      let(:dictionary_path) { directory.join("dict-e.yml") }
+      let(:event) { LogStash::Event.new("status" => "a") }
+
+      it "return the exact translation" do
+
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("no match")
+      end
+    end
+
+    context "when using a json file" do
+      let(:dictionary_path) { directory.join("dict-e.json") }
+      let(:event) { LogStash::Event.new("status" => "b") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("no match")
+      end
+    end
+
+    context "when using a csv file" do
+      let(:dictionary_path) { directory.join("dict-e.csv") }
+      let(:event) { LogStash::Event.new("status" => "c") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event.get("translation")).to eq("no match")
+      end
+    end
+  end
 end

data/spec/filters/yaml_visitor_spec.rb

@@ -0,0 +1,16 @@
+require "logstash/devutils/rspec/spec_helper"
+
+require "logstash/filters/dictionary/yaml_visitor"
+
+describe LogStash::Filters::Dictionary::YamlVisitor do
+  it 'works' do
+    yaml_string = "---\na: \n x: 3\nb: \n x: 4\n"
+    dictionary = {"c" => {"x" => 5}}
+    described_class.create.accept_with_dictionary(dictionary, Psych.parse_stream(yaml_string)).first
+    expect(dictionary.keys.sort).to eq(["a", "b", "c"])
+    values = dictionary.values
+    expect(values[0]).to eq({"x" => 5})
+    expect(values[1]).to eq({"x" => 3})
+    expect(values[2]).to eq({"x" => 4})
+  end
+end

data/spec/fixtures/regex_dict.csv

@@ -0,0 +1,4 @@
+"^2[0-9][0-9]$","OK"
+"^3[0-9][0-9]$","Redirect"
+"^4[0-9][0-9]$","Client Error"
+"^5[0-9][0-9]$","Server Error"

data/spec/fixtures/regex_union_dict.csv

@@ -0,0 +1,4 @@
+"200","OK"
+"300","Redirect"
+"400","Client Error"
+"500","Server Error"

data/spec/fixtures/tag-map-dict.yml

@@ -0,0 +1,21 @@
+---
+!!map {
+  ? !!str "eight"
+  : !!str "val-8-1|val-8-2",
+  ? !!str "five"
+  : !!str "val-5-1|val-5-2",
+  ? !!str "four"
+  : !!str "val-4-1|val-4-2",
+  ? !!str "nine"
+  : !!str "val-9-1|val-9-2",
+  ? !!str "one"
+  : !!str "val-1-1|val-1-2",
+  ? !!str "seven"
+  : !!str "val-7-1|val-7-2",
+  ? !!str "six"
+  : !!str "val-6-1|val-6-2",
+  ? !!str "three"
+  : !!str "val-3-1|val-3-2",
+  ? !!str "two"
+  : !!str "val-2-1|val-2-2",
+}

data/spec/fixtures/tag-omap-dict.yml

@@ -0,0 +1,21 @@
+---
+!!omap {
+  ? !!str "eight"
+  : !!str "val-8-1|val-8-2",
+  ? !!str "five"
+  : !!str "val-5-1|val-5-2",
+  ? !!str "four"
+  : !!str "val-4-1|val-4-2",
+  ? !!str "nine"
+  : !!str "val-9-1|val-9-2",
+  ? !!str "one"
+  : !!str "val-1-1|val-1-2",
+  ? !!str "seven"
+  : !!str "val-7-1|val-7-2",
+  ? !!str "six"
+  : !!str "val-6-1|val-6-2",
+  ? !!str "three"
+  : !!str "val-3-1|val-3-2",
+  ? !!str "two"
+  : !!str "val-2-1|val-2-2",
+}

data/spec/support/build_huge_dictionaries.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+require "securerandom"
+
+module LogStash module Filters module Dictionary
+  def self.create_huge_csv_dictionary(directory, name, size)
+    tmppath = directory.join("temp_big.csv")
+    tmppath.open("w") do |file|
+      file.puts("foo,#{SecureRandom.hex(4)}")
+      file.puts("bar,#{SecureRandom.hex(4)}")
+      size.times do |i|
+        file.puts("#{SecureRandom.hex(12)},#{1000000 + i}")
+      end
+      file.puts("baz,quux")
+    end
+    tmppath.rename(directory.join(name))
+  end
+
+  def self.create_huge_json_dictionary(directory, name, size)
+    tmppath = directory.join("temp_big.json")
+    tmppath.open("w") do |file|
+      file.puts("{")
+      file.puts('  "foo":"'.concat(SecureRandom.hex(4)).concat('",'))
+      file.puts('  "bar":"'.concat(SecureRandom.hex(4)).concat('",'))
+      size.times do |i|
+        file.puts('  "'.concat(SecureRandom.hex(12)).concat('":"').concat("#{1000000 + i}").concat('",'))
+      end
+      file.puts('  "baz":"quux"')
+      file.puts("}")
+    end
+    tmppath.rename(directory.join(name))
+  end
+end end end