logstash-filter-translate 3.1.0 → 3.2.0

data/lib/logstash/filters/single_value_update.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+
+module LogStash module Filters
+  class SingleValueUpdate
+    def initialize(field, destination, fallback, lookup)
+      @field = field
+      @destination = destination
+      @fallback = fallback
+      @use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
+      @lookup = lookup
+    end
+
+    def test_for_inclusion(event, override)
+      # Skip translation in case @destination field already exists and @override is disabled.
+      return false if event.include?(@destination) && !override
+      event.include?(@field)
+    end
+
+    def update(event)
+      # If source field is array use first value and make sure source value is string
+      source = Array(event.get(@field)).first.to_s
+      matched = [true, nil]
+      @lookup.fetch_strategy.fetch(source, matched)
+      if matched.first
+        event.set(@destination, matched.last)
+      elsif @use_fallback
+        event.set(@destination, event.sprintf(@fallback))
+        matched[0] = true
+      end
+      return matched.first
+    end
+  end
+end end

data/lib/logstash/filters/translate.rb

@@ -1,14 +1,18 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
-require "json"
-require "csv"
-
-java_import 'java.util.concurrent.locks.ReentrantReadWriteLock'
 
+require "logstash/filters/dictionary/memory"
+require "logstash/filters/dictionary/file"
+require "logstash/filters/dictionary/csv_file"
+require "logstash/filters/dictionary/yaml_file"
+require "logstash/filters/dictionary/json_file"
 
+require_relative "single_value_update"
+require_relative "array_of_values_update"
+require_relative "array_of_maps_value_update"
 # A general search and replace tool that uses a configured hash
-# and/or a file to determine replacement values. Currently supported are 
+# and/or a file to determine replacement values. Currently supported are
 # YAML, JSON, and CSV files.
 #
 # The dictionary entries can be specified in one of two ways: First,
@@ -22,21 +26,23 @@ java_import 'java.util.concurrent.locks.ReentrantReadWriteLock'
 # `regex` configuration item has been enabled), the field's value will be substituted
 # with the matched key's value from the dictionary.
 #
-# By default, the translate filter will replace the contents of the 
+# By default, the translate filter will replace the contents of the
 # maching event field (in-place). However, by using the `destination`
 # configuration item, you may also specify a target event field to
 # populate with the new translated value.
-# 
+#
 # Alternatively, for simple string search and replacements for just a few values
 # you might consider using the gsub function of the mutate filter.
-
-class LogStash::Filters::Translate < LogStash::Filters::Base
+module LogStash module Filters
+class Translate < LogStash::Filters::Base
   config_name "translate"
 
   # The name of the logstash event field containing the value to be compared for a
-  # match by the translate filter (e.g. `message`, `host`, `response_code`). 
-  # 
-  # If this field is an array, only the first value will be used.
+  # match by the translate filter (e.g. `message`, `host`, `response_code`).
+  #
+  # If this field is an array, only the first value will be used, unless
+  # you specify `iterate_on`. See below. If you want to use another element
+  # in the array then use `"[some_field][2]"`
   config :field, :validate => :string, :required => true
 
   # If the destination (or target) field already exists, this configuration item specifies
@@ -63,8 +69,8 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
 
   # The full path of the external dictionary file. The format of the table should
   # be a standard YAML, JSON or CSV with filenames ending in `.yaml`, `.yml`,
-  #`.json` or `.csv` to be read. Make sure you specify any integer-based keys in 
-  # quotes. For example, the YAML file (`.yaml` or `.yml` should look something like 
+  #`.json` or `.csv` to be read. Make sure you specify any integer-based keys in
+  # quotes. For example, the YAML file (`.yaml` or `.yml` should look something like
   # this:
   # [source,ruby]
   #     "100": Continue
@@ -88,7 +94,7 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # The destination field you wish to populate with the translated code. The default
   # is a field named `translation`. Set this to the same value as source if you want
   # to do a substitution, in this case filter will allways succeed. This will clobber
-  # the old value of the source field! 
+  # the old value of the source field!
   config :destination, :validate => :string, :default => "translation"
 
   # When `exact => true`, the translate filter will populate the destination field
@@ -108,10 +114,10 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # set to `foofing`, the destination field will be set to `barfing`.
   #
   # Set both `exact => true` AND `regex => `true` if you would like to match using dictionary
-  # keys as regular expressions. A large dictionary could be expensive to match in this case. 
+  # keys as regular expressions. A large dictionary could be expensive to match in this case.
   config :exact, :validate => :boolean, :default => true
 
-  # If you'd like to treat dictionary keys as regular expressions, set `exact => true`.
+  # If you'd like to treat dictionary keys as regular expressions, set `regex => true`.
   # Note: this is activated only when `exact => true`.
   config :regex, :validate => :boolean, :default => false
 
@@ -133,11 +139,20 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # deletes old entries on update.
   config :refresh_behaviour, :validate => ['merge', 'replace'], :default => 'merge'
 
+  # When the value that you need to perform enrichment on is a  variable sized array then specify
+  # the field name in this setting. This setting introduces two modes, 1) when the value is an
+  # array of strings and 2) when the value is an array of objects (as in JSON object).
+  # In the first mode, you should have the same field name in both `field` and `iterate_on`, the
+  # result will be an array added to the field specified in the `destination` setting. This array
+  # will have the looked up value (or the `fallback` value or nil) in same ordinal position
+  # as each sought value. In the second mode, specify the field that has the array of objects
+  # then specify the field in each object that provides the sought value with `field` and
+  # the field to write the looked up value (or the `fallback` value) to with `destination`
+  config :iterate_on, :validate => :string
+
+  attr_reader :lookup # for testing reloading
+
   def register
-    rw_lock = java.util.concurrent.locks.ReentrantReadWriteLock.new
-    @read_lock = rw_lock.readLock
-    @write_lock = rw_lock.writeLock
-    
     if @dictionary_path && !@dictionary.empty?
       raise LogStash::ConfigurationError, I18n.t(
         "logstash.agent.configuration.invalid_plugin_register",
@@ -148,153 +163,37 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
     end
 
     if @dictionary_path
-      @next_refresh = Time.now + @refresh_interval
-      raise_exception = true
-      lock_for_write { load_dictionary(raise_exception) }
+      @lookup = Dictionary::File.create(@dictionary_path, @refresh_interval, @refresh_behaviour, @exact, @regex)
+    else
+      @lookup = Dictionary::Memory.new(@dictionary, @exact, @regex)
+    end
+    if @iterate_on.nil?
+      @updater = SingleValueUpdate.new(@field, @destination, @fallback, @lookup)
+    elsif @iterate_on == @field
+      @updater = ArrayOfValuesUpdate.new(@iterate_on, @destination, @fallback, @lookup)
+    else
+      @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @field, @destination, @fallback, @lookup)
     end
 
-    @logger.debug? and @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @dictionary)
+    @logger.debug? && @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
     if @exact
-      @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
+      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
     else
-      @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
+      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
     end
   end # def register
 
-  def lock_for_read
-    @read_lock.lock
-    begin
-      yield
-    ensure
-      @read_lock.unlock
-    end
-  end
-
-  def lock_for_write
-    @write_lock.lock
-    begin
-      yield
-    ensure
-      @write_lock.unlock
-    end
+  def close
+    @lookup.stop_scheduler
   end
 
   def filter(event)
-    if @dictionary_path
-      if needs_refresh?
-        lock_for_write do
-          if needs_refresh?
-            load_dictionary
-            @next_refresh = Time.now + @refresh_interval
-            @logger.info("refreshing dictionary file")
-          end
-        end
-      end
-    end
-
-    return unless event.include?(@field) # Skip translation in case event does not have @event field.
-    return if event.include?(@destination) and not @override # Skip translation in case @destination field already exists and @override is disabled.
-
+    return unless @updater.test_for_inclusion(event, @override)
     begin
-      #If source field is array use first value and make sure source value is string
-      source = event.get(@field).is_a?(Array) ? event.get(@field).first.to_s : event.get(@field).to_s
-      matched = false
-      if @exact
-        if @regex
-          key = @dictionary.keys.detect{|k| source.match(Regexp.new(k))}
-          if key
-            event.set(@destination, lock_for_read { @dictionary[key] })
-            matched = true
-          end
-        elsif @dictionary.include?(source)
-          event.set(@destination, lock_for_read { @dictionary[source] })
-          matched = true
-        end
-      else
-        translation = lock_for_read { source.gsub(Regexp.union(@dictionary.keys), @dictionary) }
-        
-        if source != translation
-          event.set(@destination, translation.force_encoding(Encoding::UTF_8))
-          matched = true
-        end
-      end
-
-      if not matched and @fallback
-        event.set(@destination, event.sprintf(@fallback))
-        matched = true
-      end
-      filter_matched(event) if matched or @field == @destination
+      filter_matched(event) if @updater.update(event) || @field == @destination
     rescue Exception => e
       @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
     end
   end # def filter
-
-  private
-
-  def load_dictionary(raise_exception=false)
-    if /.y[a]?ml$/.match(@dictionary_path)
-      load_yaml(raise_exception)
-    elsif @dictionary_path.end_with?(".json")
-      load_json(raise_exception)
-    elsif @dictionary_path.end_with?(".csv")
-      load_csv(raise_exception)
-    else
-      raise "#{self.class.name}: Dictionary #{@dictionary_path} have a non valid format"
-    end
-  rescue => e
-    loading_exception(e, raise_exception)
-  end
-
-  def load_yaml(raise_exception=false)
-    if !File.exists?(@dictionary_path)
-      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
-      return
-    end
-    refresh_dictionary!(YAML.load_file(@dictionary_path))
-  end
-
-  def load_json(raise_exception=false)
-    if !File.exists?(@dictionary_path)
-      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
-      return
-    end
-    refresh_dictionary!(JSON.parse(File.read(@dictionary_path)))
-  end
-
-  def load_csv(raise_exception=false)
-    if !File.exists?(@dictionary_path)
-      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
-      return
-    end
-    data = CSV.read(@dictionary_path).inject(Hash.new) do |acc, v|
-      acc[v[0]] = v[1]
-      acc
-    end
-    refresh_dictionary!(data)
-  end
-
-  def refresh_dictionary!(data)
-    case @refresh_behaviour
-    when 'merge'
-     @dictionary.merge!(data)
-    when 'replace'
-     @dictionary = data
-    else
-     # we really should never get here
-     raise(LogStash::ConfigurationError, "Unknown value for refresh_behaviour=#{@refresh_behaviour.to_s}")
-    end
-  end
-
-  def loading_exception(e, raise_exception=false)
-    msg = "#{self.class.name}: #{e.message} when loading dictionary file at #{@dictionary_path}"
-    if raise_exception
-      raise RuntimeError.new(msg)
-    else
-      @logger.warn("#{msg}, continuing with old dictionary", :dictionary_path => @dictionary_path)
-    end
-  end
-
-  def needs_refresh?
-    @next_refresh < Time.now
-  end
 end # class LogStash::Filters::Translate
+end end

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.1.0'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,11 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'rufus-scheduler'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'rspec-sequencing'
+  s.add_development_dependency "rspec-wait"
+  s.add_development_dependency "benchmark-ips"
 end
 

data/spec/filters/benchmark_rspec.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+
+require "logstash/filters/translate"
+require "benchmark/ips"
+
+module BenchmarkingFileBuilder
+  def self.create_huge_csv_dictionary(directory, name, size)
+    tmppath = directory.join("temp_big.csv")
+    tmppath.open("w") do |file|
+      file.puts("foo,#{SecureRandom.hex(4)}")
+      file.puts("bar,#{SecureRandom.hex(4)}")
+      size.times do |i|
+        file.puts("#{SecureRandom.hex(12)},#{1000000 + i}")
+      end
+      file.puts("baz,quux")
+    end
+    tmppath.rename(directory.join(name))
+  end
+end
+
+describe LogStash::Filters::Translate do
+  let(:directory) { Pathname.new(Stud::Temporary.directory) }
+  let(:dictionary_name) { "dict-h.csv" }
+  let(:dictionary_path) { directory.join(dictionary_name) }
+  let(:dictionary_size) { 100000 }
+  let(:config) do
+    {
+      "field"       => "[status]",
+      "destination" => "[translation]",
+      "dictionary_path"  => dictionary_path.to_path,
+      "exact"       => true,
+      "regex"       => false,
+      "refresh_interval" => 0,
+      "override" => true,
+      "refresh_behaviour" => "merge"
+    }
+  end
+  before do
+    directory
+    BenchmarkingFileBuilder.create_huge_csv_dictionary(directory, dictionary_name, dictionary_size)
+  end
+
+  it 'how fast is the filter method?' do
+    plugin = described_class.new(config)
+    plugin.register
+    event = LogStash::Event.new("status" => "baz", "translation" => "foo")
+
+    Benchmark.ips do |x|
+      x.config(:time => 20, :warmup => 120)
+      x.report("filter(event)") { plugin.filter(event) }
+    end
+    expect(event.get("[translation]")).to eq("quux")
+  end
+
+  it 'how fast is the new, register then filter method?' do
+    event = LogStash::Event.new("status" => "baz", "translation" => "foo")
+
+    Benchmark.ips do |x|
+      x.config(:time => 10, :warmup => 120)
+      x.report("new, register, filter(event)") do
+        plugin = described_class.new(config)
+        plugin.register
+        plugin.filter(event)
+       end
+    end
+    expect(event.get("[translation]")).to eq("quux")
+  end
+end

data/spec/filters/scheduling_spec.rb

@@ -0,0 +1,200 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "support/rspec_wait_handler_helper"
+require "support/build_huge_dictionaries"
+
+require "rspec_sequencing"
+
+require "logstash/filters/translate"
+
+describe LogStash::Filters::Translate do
+  let(:directory) { Pathname.new(Stud::Temporary.directory) }
+  describe "scheduled reloading" do
+    subject { described_class.new(config) }
+
+    let(:config) do
+      {
+        "field"       => "[status]",
+        "destination" => "[translation]",
+        "dictionary_path"  => dictionary_path.to_path,
+        "exact"       => true,
+        "regex"       => false,
+        "refresh_interval" => 1,
+        "override" => true,
+        "refresh_behaviour" => refresh_behaviour
+      }
+    end
+
+    let(:event) { LogStash::Event.new("status" => "b") }
+
+    before do
+      directory
+      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      dictionary_path.open("wb") do |file|
+        file.puts("a,1\nb,2\nc,3\n")
+      end
+      subject.register
+    end
+
+    after do
+      FileUtils.rm_rf(directory)
+      wait(1.0).for{Dir.exist?(directory)}.to eq(false)
+    end
+
+    context "replace" do
+      let(:dictionary_path) { directory.join("dict-r.csv") }
+      let(:refresh_behaviour) { "replace" }
+      let(:actions) do
+        RSpec::Sequencing
+          .run("translate") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("2"), "field [translation] did not eq '2'"
+          end
+          .then_after(1,"modify file") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,11\nb,12\nc,13\n")
+            end
+          end
+          .then_after(1.2, "wait then translate again") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("12"), "field [translation] did not eq '12'"
+          end
+          .then("stop") do
+            subject.close
+          end
+      end
+
+      it "updates the event after scheduled reload" do
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+    end
+
+    context "merge" do
+      let(:dictionary_path) { directory.join("dict-m.csv") }
+      let(:refresh_behaviour) { "merge" }
+      let(:actions) do
+        RSpec::Sequencing
+          .run("translate") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("2"), "field [translation] did not eq '2'"
+          end
+          .then_after(1,"modify file") do
+            dictionary_path.open("w") do |file|
+              file.puts("a,21\nb,22\nc,23\n")
+            end
+          end
+          .then_after(1.2, "wait then translate again") do
+            subject.filter(event)
+            wait(0.1).for{event.get("[translation]")}.to eq("22"), "field [translation] did not eq '22'"
+          end
+          .then("stop") do
+            subject.close
+          end
+      end
+
+      it "updates the event after scheduled reload" do
+        actions.activate_quietly
+        actions.assert_no_errors
+      end
+    end
+  end
+
+  describe "huge json file merge" do
+    let(:dictionary_path) { directory.join("dict-h.json") }
+    let(:dictionary_size) { 100000 }
+    let(:config) do
+      {
+        "field"       => "[status]",
+        "destination" => "[translation]",
+        "dictionary_path"  => dictionary_path.to_path,
+        "exact"       => true,
+        "regex"       => false,
+        "refresh_interval" => 1,
+        "override" => true,
+        "refresh_behaviour" => "merge"
+      }
+    end
+    let(:event) { LogStash::Event.new("status" => "baz", "translation" => "foo") }
+    subject { described_class.new(config) }
+
+    before do
+      directory
+      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      LogStash::Filters::Dictionary.create_huge_json_dictionary(directory, "dict-h.json", dictionary_size)
+      subject.register
+    end
+
+    let(:actions) do
+      RSpec::Sequencing
+        .run("translate") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then_after(0.1,"modify file") do
+          LogStash::Filters::Dictionary.create_huge_json_dictionary(directory, "dict-h.json", dictionary_size)
+        end
+        .then_after(1.8, "wait then translate again") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then("stop") do
+          subject.close
+        end
+    end
+
+    it "updates the event after scheduled reload" do
+      actions.activate_quietly
+      actions.assert_no_errors
+    end
+  end
+
+  describe "huge csv file merge" do
+    let(:dictionary_path) { directory.join("dict-h.csv") }
+    let(:dictionary_size) { 100000 }
+    let(:config) do
+      {
+        "field"       => "[status]",
+        "destination" => "[translation]",
+        "dictionary_path"  => dictionary_path.to_path,
+        "exact"       => true,
+        "regex"       => false,
+        "refresh_interval" => 1,
+        "override" => true,
+        "refresh_behaviour" => "merge"
+      }
+    end
+    let(:event) { LogStash::Event.new("status" => "bar", "translation" => "foo") }
+    subject { described_class.new(config) }
+
+    before do
+      directory
+      wait(1.0).for{Dir.exist?(directory)}.to eq(true)
+      LogStash::Filters::Dictionary.create_huge_csv_dictionary(directory, "dict-h.csv", dictionary_size)
+      subject.register
+    end
+
+    let(:actions) do
+      RSpec::Sequencing
+        .run("translate") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then_after(0.1,"modify file") do
+          LogStash::Filters::Dictionary.create_huge_csv_dictionary(directory, "dict-h.csv", dictionary_size)
+        end
+        .then_after(1.8, "wait then translate again") do
+          subject.filter(event)
+          wait(0.1).for{event.get("[translation]")}.not_to eq("foo"), "field [translation] should not be 'foo'"
+        end
+        .then("stop") do
+          subject.close
+        end
+    end
+
+    it "updates the event after scheduled reload" do
+      actions.activate_quietly
+      actions.assert_no_errors
+    end
+  end
+end