logstash-filter-translate 2.0.2 → 2.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5522423e565a186cb510bff658b46dc95a6faafa
-  data.tar.gz: 6a56306cc43c2e35376372f0a48283f8217030cc
+  metadata.gz: 8003929e0ef9c30f41f95b144ce3872b60cceaa7
+  data.tar.gz: b92c98f43df88c648a684f638cd04fed8e12336b
 SHA512:
-  metadata.gz: db2cbcff7dad08c6b5f84fbf59f6400c734da6d723a6b453d36c7e5ca88814f55e26cb0100fe7989e7a7f4f29358bed8dcab4bc670982c993329a05e7566ee34
-  data.tar.gz: 962fb927bd78ef64b90f0de95cef9042bf06be1c489af1c0901fa9a2ab7eb517dd9aa11f642a79fb1235d0e625e7fcd024d873e88041c1d9f4f31cde1093027f
+  metadata.gz: 2a65ab7595ecbd6eb16bbec0127f9052060c6601be096658b76b9ccdbb4ede9b176047adae3c9aaeeb8e17d22d6cf7f54a8bd6ca00bed52052c40b8f95494cbc
+  data.tar.gz: 0a7bb3d83cebf55531bf35bff3a626528932fe78de627e46ac0edc91fa7230296069943503529fec868f942a79dd914c74492550fd658ce69f98df98df458c51

data/CHANGELOG.md

@@ -1,7 +1,20 @@
+# 2.1.3
+  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+# 2.1.2
+  - New dependency requirements for logstash-core for the 5.0 release
+## 2.1.1
+  - Add more descriptive message with the dictionary could not be loaded,
+  also include test for it. 
+
+## 2.1.0
+  - Added other formats, a part from YAML, to be used when loading
+  dictionaries from files in this plugin. Current supported formats are
+  YAML, JSON and CSV.
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
-   instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
- - Dependency on logstash-core update to 2.0
+  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+  instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
+  - Dependency on logstash-core update to 2.0
 
-# 0.1.10
+## 0.1.10
   - fix failing test due to a missing encoding: utf8 magic header

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-translate-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-translate-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/translate.rb

@@ -1,13 +1,16 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
+require "json"
+require "csv"
 
 # A general search and replace tool which uses a configured hash
-# and/or a YAML file to determine replacement values.
+# and/or a file to determine replacement values. Currently supported are 
+# YAML, JSON and CSV files.
 #
 # The dictionary entries can be specified in one of two ways: First,
 # the `dictionary` configuration item may contain a hash representing
-# the mapping. Second, an external YAML file (readable by logstash) may be specified
+# the mapping. Second, an external file (readable by logstash) may be specified
 # in the `dictionary_path` configuration item. These two methods may not be used
 # in conjunction; it will produce an error.
 #
@@ -39,7 +42,7 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   config :override, :validate => :boolean, :default => false
 
   # The dictionary to use for translation, when specified in the logstash filter
-  # configuration item (i.e. do not use the `@dictionary_path` YAML file)
+  # configuration item (i.e. do not use the `@dictionary_path` file)
   # Example:
   # [source,ruby]
   #     filter {
@@ -53,22 +56,27 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # NOTE: it is an error to specify both `dictionary` and `dictionary_path`
   config :dictionary, :validate => :hash,  :default => {}
 
-  # The full path of the external YAML dictionary file. The format of the table
-  # should be a standard YAML file. Make sure you specify any integer-based keys
-  # in quotes. The YAML file should look something like this:
+  # The full path of the external dictionary file. The format of the table
+  # should be a standard YAML, JSON or CSV. Make sure you specify any integer-based keys
+  # in quotes. For example, the YAML file should look something like this:
   # [source,ruby]
   #     "100": Continue
   #     "101": Switching Protocols
   #     merci: gracias
   #     old version: new version
-  #     
+  #
   # NOTE: it is an error to specify both `dictionary` and `dictionary_path`
+  # NOTE: Currently supported formats are YAML, JSON and CSV, format selection is
+  # based on the file extension, json for JSON, (yaml|yml) for YAML and csv for CSV.
+  # NOTE: The JSON format only supports simple key/value, unnested objects. The CSV
+  # format expects exactly two columns with the first serving as the original text,
+  # the second column as the replacement
   config :dictionary_path, :validate => :path
 
   # When using a dictionary file, this setting will indicate how frequently
-  # (in seconds) logstash will check the YAML file for updates.
+  # (in seconds) logstash will check the dictionary file for updates.
   config :refresh_interval, :validate => :number, :default => 300
-  
+
   # The destination field you wish to populate with the translated code. The default
   # is a field named `translation`. Set this to the same value as source if you want
   # to do a substitution, in this case filter will allways succeed. This will clobber
@@ -86,7 +94,7 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   #
   # If logstash receives an event with the `data` field set to `foo`, and `exact => true`,
   # the destination field will be populated with the string `bar`.
-  
+
   # If `exact => false`, and logstash receives the same event, the destination field
   # will be also set to `bar`. However, if logstash receives an event with the `data` field
   # set to `foofing`, the destination field will be set to `barfing`.
@@ -112,14 +120,13 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
   config :fallback, :validate => :string
 
-  public
   def register
     if @dictionary_path
       @next_refresh = Time.now + @refresh_interval
-      registering = true
-      load_yaml(registering)
+      raise_exception = true
+      load_dictionary(raise_exception)
     end
-    
+
     @logger.debug? and @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @dictionary)
     if @exact
       @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
@@ -128,36 +135,15 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
     end
   end # def register
 
-  public
-  def load_yaml(registering=false)
-    if !File.exists?(@dictionary_path)
-      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
-      return
-    end
-
-    begin
-      @dictionary.merge!(YAML.load_file(@dictionary_path))
-    rescue Exception => e
-      if registering
-        raise "#{self.class.name}: Bad Syntax in dictionary file #{@dictionary_path}"
-      else
-        @logger.warn("#{self.class.name}: Bad Syntax in dictionary file, continuing with old dictionary", :dictionary_path => @dictionary_path)
-      end
-    end
-  end
-
-  public
   def filter(event)
-    
-
     if @dictionary_path
       if @next_refresh < Time.now
-        load_yaml
+        load_dictionary
         @next_refresh = Time.now + @refresh_interval
         @logger.info("refreshing dictionary file")
       end
     end
-    
+
     return unless event.include?(@field) # Skip translation in case event does not have @event field.
     return if event.include?(@destination) and not @override # Skip translation in case @destination field already exists and @override is disabled.
 
@@ -193,4 +179,61 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
       @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
     end
   end # def filter
+
+  private
+
+  def load_dictionary(raise_exception=false)
+    if /.y[a]?ml$/.match(@dictionary_path)
+      load_yaml(raise_exception)
+    elsif @dictionary_path.end_with?(".json")
+      load_json(raise_exception)
+    elsif @dictionary_path.end_with?(".csv")
+      load_csv(raise_exception)
+    else
+      raise "#{self.class.name}: Dictionary #{@dictionary_path} have a non valid format"
+    end
+  rescue => e
+    loading_exception(e, raise_exception)
+  end
+
+  def load_yaml(raise_exception=false)
+    if !File.exists?(@dictionary_path)
+      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
+      return
+    end
+    merge_dictionary!(YAML.load_file(@dictionary_path), raise_exception)
+  end
+
+  def load_json(raise_exception=false)
+    if !File.exists?(@dictionary_path)
+      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
+      return
+    end
+    merge_dictionary!(JSON.parse(File.read(@dictionary_path)), raise_exception)
+  end
+
+  def load_csv(raise_exception=false)
+    if !File.exists?(@dictionary_path)
+      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
+      return
+    end
+    data = CSV.read(@dictionary_path).inject(Hash.new) do |acc, v|
+      acc[v[0]] = v[1]
+      acc
+    end
+    merge_dictionary!(data, raise_exception)
+  end
+
+  def merge_dictionary!(data, raise_exception=false)
+      @dictionary.merge!(data)
+  end
+
+  def loading_exception(e, raise_exception=false)
+    msg = "#{self.class.name}: #{e.message} when loading dictionary file at #{@dictionary_path}"
+    if raise_exception
+      raise RuntimeError.new(msg)
+    else
+      @logger.warn("#{msg}, continuing with old dictionary", :dictionary_path => @dictionary_path)
+    end
+  end
 end # class LogStash::Filters::Translate

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '2.0.2'
+  s.version         = '2.1.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "A general search and replace tool which uses a configured hash and/or a YAML file to determine replacement values."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/translate_spec.rb

@@ -120,4 +120,64 @@ describe LogStash::Filters::Translate do
     end
   end
 
+  describe "loading a dictionary" do
+
+    let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict-wrong.yml") }
+
+    let(:config) do
+      {
+        "field"       => "status",
+        "destination" => "translation",
+        "dictionary_path"  => dictionary_path,
+        "exact"       => true,
+        "regex"       => false
+      }
+    end
+
+    it "raises exception when loading" do
+      error = "(#{dictionary_path}): mapping values are not allowed here at line 1 column 45 when loading dictionary file at #{dictionary_path}"
+      expect { subject.register }.to raise_error("#{described_class}: #{error}")
+    end
+
+    context "when using a yml file" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.yml") }
+      let(:event) { LogStash::Event.new("status" => "a") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event["translation"]).to eq(1)
+      end
+    end
+
+    context "when using a json file" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.json") }
+      let(:event) { LogStash::Event.new("status" => "b") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event["translation"]).to eq(20)
+      end
+    end
+
+    context "when using a csv file" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.csv") }
+      let(:event) { LogStash::Event.new("status" => "c") }
+
+      it "return the exact translation" do
+        subject.register
+        subject.filter(event)
+        expect(event["translation"]).to eq("300")
+      end
+    end
+
+    context "when using an uknown file" do
+      let(:dictionary_path)  { File.join(File.dirname(__FILE__), "..", "fixtures", "dict.other") }
+
+      it "return the exact translation" do
+        expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} have a non valid format/)
+      end
+    end
+  end
 end

data/spec/fixtures/dict-wrong.yml

@@ -0,0 +1 @@
+foo: somebody said I should put a colon here: so I did

data/spec/fixtures/dict.csv

@@ -0,0 +1,3 @@
+"a",100
+"b",200
+"c",300

data/spec/fixtures/dict.json

@@ -0,0 +1,5 @@
+{
+  "a" : 10,
+  "b" : 20,
+  "c" : 30
+}

data/spec/fixtures/dict.other

@@ -0,0 +1,3 @@
+"a" | 400
+"b" | 500
+"c" | 600

data/spec/fixtures/dict.yml

@@ -0,0 +1,3 @@
+a : 1
+b : 2
+c : 3

metadata

@@ -1,39 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-translate
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
-      - !ruby/object:Gem::Version
-        version: 3.0.0
-  name: logstash-core
+        version: '1.0'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.0.0.beta2
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -41,7 +35,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -59,6 +53,11 @@ files:
 - lib/logstash/filters/translate.rb
 - logstash-filter-translate.gemspec
 - spec/filters/translate_spec.rb
+- spec/fixtures/dict-wrong.yml
+- spec/fixtures/dict.csv
+- spec/fixtures/dict.json
+- spec/fixtures/dict.other
+- spec/fixtures/dict.yml
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -71,12 +70,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -87,3 +86,8 @@ specification_version: 4
 summary: A general search and replace tool which uses a configured hash and/or a YAML file to determine replacement values.
 test_files:
 - spec/filters/translate_spec.rb
+- spec/fixtures/dict-wrong.yml
+- spec/fixtures/dict.csv
+- spec/fixtures/dict.json
+- spec/fixtures/dict.other
+- spec/fixtures/dict.yml