logstash-filter-translate 3.0.4 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89b8f50e5b8c125eb357a4675e6f8ff2f85899111244f70f12c78748e25c3252
-  data.tar.gz: c7891499c846ec33ca21d8b7c5c24a16321c62c9c38952aed67ee0fb5aabf847
+  metadata.gz: 5e4b8c724a742d47aa89a6237e127be569f2675a19c5752e719918bff8489c77
+  data.tar.gz: 406d7e0417251e10d1142d19027af9557f9a0993782bb16780bcdc86c943bd5e
 SHA512:
-  metadata.gz: c449a8bb6bdeb443264a551bfc87c41b099c5c659fc1c2ea120619cf65bc71a80d9c4c037737e238b724d121f6572454e5ed222fe95400cff4e037ea2d64042f
-  data.tar.gz: 20b05d693382032ee6718a57cce20e398c590cf3464bb713b86fd0192183e2026443c63896df20d43de53304f1e68eb11e6f59d21ad67f8d6b9f572766f296de
+  metadata.gz: e25757bc66a2b1afa15318161c6a8182f5258e9a8f395c870e0826224f6111bfa607950e678058a423aecb733f9f23edbbaed7778067f239269786afc76341d9
+  data.tar.gz: c84f492fad3418db7e18333658691fe5a8109c6dfc15589e88415198d348cce4fa05d4b966762072387441dec06210d6c7bf5fcc92fe5bec94ffe1b5026eba9d

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.0
+  - Add 'refresh_behaviour' to either 'merge' or 'replace' during a refresh #57
+
 ## 3.0.4
   - Update gemspec summary
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -60,6 +60,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-override>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-refresh_interval>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-regex>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-refresh_behaviour>> |<<string,string>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -91,10 +92,12 @@ Example:
 [source,ruby]
     filter {
       translate {
-        dictionary => [ "100", "Continue",
-                        "101", "Switching Protocols",
-                        "merci", "thank you",
-                        "old version", "new version" ]
+        dictionary => { 
+          "100"         => "Continue",
+          "101"         => "Switching Protocols",
+          "merci"       => "thank you",
+          "old version" => "new version"
+        }
       }
     }
 
@@ -205,7 +208,17 @@ When using a dictionary file, this setting will indicate how frequently
 If you'd like to treat dictionary keys as regular expressions, set `exact => true`.
 Note: this is activated only when `exact => true`.
 
+[id="plugins-{type}s-{plugin}-refresh_behaviour"]
+===== `refresh_behaviour`
+
+  * Value type is <<string,string>>
+  * Default value is `merge`
+
+When using a dictionary file, this setting indicates how the update will be executed.
+Setting this to `merge` leads to entries removed from the dictionary file being kept;
+`replace` deletes old entries on update.
+
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/translate.rb

@@ -61,9 +61,11 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # NOTE: It is an error to specify both `dictionary` and `dictionary_path`.
   config :dictionary, :validate => :hash,  :default => {}
 
-  # The full path of the external dictionary file. The format of the table
-  # should be a standard YAML, JSON, or CSV. Make sure you specify any integer-based keys
-  # in quotes. For example, the YAML file should look something like this:
+  # The full path of the external dictionary file. The format of the table should
+  # be a standard YAML, JSON or CSV with filenames ending in `.yaml`, `.yml`,
+  #`.json` or `.csv` to be read. Make sure you specify any integer-based keys in 
+  # quotes. For example, the YAML file (`.yaml` or `.yml` should look something like 
+  # this:
   # [source,ruby]
   #     "100": Continue
   #     "101": Switching Protocols
@@ -126,6 +128,11 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
   config :fallback, :validate => :string
 
+  # When using a dictionary file, this setting indicates how the update will be executed.
+  # Setting this to `merge` leads to entries removed from the dictionary file being kept; `replace`
+  # deletes old entries on update.
+  config :refresh_behaviour, :validate => ['merge', 'replace'], :default => 'merge'
+
   def register
     rw_lock = java.util.concurrent.locks.ReentrantReadWriteLock.new
     @read_lock = rw_lock.readLock
@@ -243,7 +250,7 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
       @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
       return
     end
-    merge_dictionary!(YAML.load_file(@dictionary_path), raise_exception)
+    refresh_dictionary!(YAML.load_file(@dictionary_path))
   end
 
   def load_json(raise_exception=false)
@@ -251,7 +258,7 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
       @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
       return
     end
-    merge_dictionary!(JSON.parse(File.read(@dictionary_path)), raise_exception)
+    refresh_dictionary!(JSON.parse(File.read(@dictionary_path)))
   end
 
   def load_csv(raise_exception=false)
@@ -263,11 +270,19 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
       acc[v[0]] = v[1]
       acc
     end
-    merge_dictionary!(data, raise_exception)
+    refresh_dictionary!(data)
   end
 
-  def merge_dictionary!(data, raise_exception=false)
-    @dictionary.merge!(data)
+  def refresh_dictionary!(data)
+    case @refresh_behaviour
+    when 'merge'
+     @dictionary.merge!(data)
+    when 'replace'
+     @dictionary = data
+    else
+     # we really should never get here
+     raise(LogStash::ConfigurationError, "Unknown value for refresh_behaviour=#{@refresh_behaviour.to_s}")
+    end
   end
 
   def loading_exception(e, raise_exception=false)

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.0.4'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/translate_spec.rb

@@ -195,4 +195,73 @@ describe LogStash::Filters::Translate do
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end
+
+  describe "refresh_behaviour" do
+    let(:dictionary_content) { "a : 1\nb : 2\nc : 3" }
+    let(:modified_content) { "a : 1\nb : 4" }
+    let(:dictionary_path)  { "#{Stud::Temporary.pathname}.yml" }
+    let(:refresh_behaviour) { "merge" }
+    let(:config) do
+      {
+        "field" => "status",
+        "destination" => "translation",
+        "dictionary_path" => dictionary_path,
+        "refresh_interval" => 10000, # we're controlling this manually
+        "exact" => true,
+        "regex" => false,
+        "fallback" => "no match",
+        "refresh_behaviour" => refresh_behaviour
+      }
+    end
+
+    before :each do
+      IO.write(dictionary_path, dictionary_content)
+      subject.register
+    end
+
+    let(:before_mod) { LogStash::Event.new("status" => "b") }
+    let(:after_mod) { LogStash::Event.new("status" => "b") }
+    let(:before_del) { LogStash::Event.new("status" => "c") }
+    let(:after_del) { LogStash::Event.new("status" => "c") }
+
+    context "when 'merge'" do
+      let(:refresh_behaviour) { 'merge' }
+      it "overwrites existing entries" do
+        subject.filter(before_mod)
+        IO.write(dictionary_path, modified_content)
+        subject.send(:load_dictionary)
+        subject.filter(after_mod)
+        expect(before_mod.get("translation")).to eq(2)
+        expect(after_mod.get("translation")).to eq(4)
+      end
+      it "keeps leftover entries" do
+        subject.filter(before_del)
+        IO.write(dictionary_path, modified_content)
+        subject.send(:load_dictionary)
+        subject.filter(after_del)
+        expect(before_del.get("translation")).to eq(3)
+        expect(after_del.get("translation")).to eq(3)
+      end
+    end
+
+    context "when 'replace'" do
+      let(:refresh_behaviour) { 'replace' }
+      it "overwrites existing entries" do
+        subject.filter(before_mod)
+        IO.write(dictionary_path, modified_content)
+        subject.send(:load_dictionary)
+        subject.filter(after_mod)
+        expect(before_mod.get("translation")).to eq(2)
+        expect(after_mod.get("translation")).to eq(4)
+      end
+      it "removes leftover entries" do
+        subject.filter(before_del)
+        IO.write(dictionary_path, modified_content)
+        subject.send(:load_dictionary)
+        subject.filter(after_del)
+        expect(before_del.get("translation")).to eq(3)
+        expect(after_del.get("translation")).to eq("no match")
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-translate
 version: !ruby/object:Gem::Version
-  version: 3.0.4
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -89,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Replaces field contents based on a hash or YAML file