logstash-filter-translate 3.1.0 → 3.3.0

data/lib/logstash/filters/fetch_strategy/file.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+
+module LogStash module Filters module FetchStrategy module File
+  class Exact
+    def initialize(dictionary, rw_lock)
+      @dictionary = dictionary
+      @read_lock = rw_lock.readLock
+    end
+
+    def dictionary_updated
+    end
+
+    def fetch(source, results)
+      @read_lock.lock
+      begin
+        if @dictionary.include?(source)
+          results[1] = LogStash::Util.deep_clone(@dictionary[source])
+        else
+          results[0] = false
+        end
+      ensure
+        @read_lock.unlock
+      end
+    end
+  end
+
+  class ExactRegex
+    def initialize(dictionary, rw_lock)
+      @keys_regex = Hash.new()
+      @dictionary = dictionary
+      @read_lock = rw_lock.readLock
+    end
+
+    def dictionary_updated
+      @keys_regex.clear
+      # rebuilding the regex map is time expensive
+      # 100 000 keys takes 0.5 seconds on a high spec Macbook Pro
+      # at least we are not doing it for every event like before
+      @dictionary.keys.each{|k| @keys_regex[k] = Regexp.new(k)}
+    end
+
+    def fetch(source, results)
+      @read_lock.lock
+      begin
+        key = @dictionary.keys.detect{|k| source.match(@keys_regex[k])}
+        if key.nil?
+          results[0] = false
+        else
+          results[1] = LogStash::Util.deep_clone(@dictionary[key])
+        end
+      ensure
+        @read_lock.unlock
+      end
+    end
+  end
+
+  class RegexUnion
+    def initialize(dictionary, rw_lock)
+      @dictionary = dictionary
+      @read_lock = rw_lock.readLock
+    end
+
+    def dictionary_updated
+      @union_regex_keys = Regexp.union(@dictionary.keys)
+    end
+
+    def fetch(source, results)
+      @read_lock.lock
+      begin
+        value = source.gsub(@union_regex_keys, @dictionary)
+        if source == value
+          results[0] = false
+        else
+          results[1] = LogStash::Util.deep_clone(value)
+        end
+      ensure
+        @read_lock.unlock
+      end
+    end
+  end
+end end end end

data/lib/logstash/filters/fetch_strategy/memory.rb

@@ -0,0 +1,52 @@
+# encoding: utf-8
+
+module LogStash module Filters module FetchStrategy module Memory
+  class Exact
+    def initialize(dictionary)
+      @dictionary = dictionary
+    end
+
+    def fetch(source, results)
+      if @dictionary.include?(source)
+        results[1] = LogStash::Util.deep_clone(@dictionary[source])
+      else
+        results[0] = false
+      end
+    end
+  end
+
+  class ExactRegex
+    def initialize(dictionary)
+      @keys_regex = Hash.new()
+      @dictionary = dictionary
+      @dictionary.keys.each{|k| @keys_regex[k] = Regexp.new(k)}
+    end
+
+    def fetch(source, results)
+      key = @dictionary.keys.detect{|k| source.match(@keys_regex[k])}
+      if key.nil?
+        results[0] = false
+      else
+        results[1] = LogStash::Util.deep_clone(@dictionary[key])
+      end
+    end
+  end
+
+  class RegexUnion
+    def initialize(dictionary)
+      @dictionary = dictionary
+      @union_regex_keys = Regexp.union(@dictionary.keys)
+    end
+
+    def fetch(source, results)
+      value = source.gsub(@union_regex_keys, @dictionary)
+      if source == value
+        results[0] = false
+      else
+        results[1] = LogStash::Util.deep_clone(value)
+      end
+    end
+  end
+end end end end
+
+

data/lib/logstash/filters/single_value_update.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+module LogStash module Filters
+  class SingleValueUpdate
+    class CoerceString
+      def call(source) source; end
+    end
+    class CoerceArray
+      def call(source) source.first.to_s; end
+    end
+    class CoerceOther
+      def call(source) source.to_s end
+    end
+
+    def initialize(field, destination, fallback, lookup)
+      @field = field
+      @destination = destination
+      @fallback = fallback
+      @use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
+      @lookup = lookup
+      @coercers_table = {}
+      @coercers_table.default = CoerceOther.new
+      @coercers_table[String] = CoerceString.new
+      @coercers_table[Array] = CoerceArray.new
+    end
+
+    def test_for_inclusion(event, override)
+      # Skip translation in case @destination field already exists and @override is disabled.
+      return false if !override && event.include?(@destination)
+      event.include?(@field)
+    end
+
+    def update(event)
+      # If source field is array use first value and make sure source value is string
+      # source = Array(event.get(@field)).first.to_s
+      source = event.get(@field)
+      source = @coercers_table[source.class].call(source)
+      matched = [true, nil]
+      @lookup.fetch_strategy.fetch(source, matched)
+      if matched.first
+        event.set(@destination, matched.last)
+      elsif @use_fallback
+        event.set(@destination, event.sprintf(@fallback))
+        matched[0] = true
+      end
+      return matched.first
+    end
+  end
+end end

data/lib/logstash/filters/translate.rb

@@ -1,14 +1,21 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
-require "json"
-require "csv"
-
-java_import 'java.util.concurrent.locks.ReentrantReadWriteLock'
-
-
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+require 'logstash/plugin_mixins/deprecation_logger_support'
+
+require "logstash/filters/dictionary/memory"
+require "logstash/filters/dictionary/file"
+require "logstash/filters/dictionary/csv_file"
+require "logstash/filters/dictionary/yaml_file"
+require "logstash/filters/dictionary/json_file"
+
+require_relative "single_value_update"
+require_relative "array_of_values_update"
+require_relative "array_of_maps_value_update"
 # A general search and replace tool that uses a configured hash
-# and/or a file to determine replacement values. Currently supported are 
+# and/or a file to determine replacement values. Currently supported are
 # YAML, JSON, and CSV files.
 #
 # The dictionary entries can be specified in one of two ways: First,
@@ -22,27 +29,38 @@ java_import 'java.util.concurrent.locks.ReentrantReadWriteLock'
 # `regex` configuration item has been enabled), the field's value will be substituted
 # with the matched key's value from the dictionary.
 #
-# By default, the translate filter will replace the contents of the 
+# By default, the translate filter will replace the contents of the
 # maching event field (in-place). However, by using the `destination`
 # configuration item, you may also specify a target event field to
 # populate with the new translated value.
-# 
+#
 # Alternatively, for simple string search and replacements for just a few values
 # you might consider using the gsub function of the mutate filter.
+module LogStash module Filters
+class Translate < LogStash::Filters::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::DeprecationLoggerSupport
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
 
-class LogStash::Filters::Translate < LogStash::Filters::Base
   config_name "translate"
 
   # The name of the logstash event field containing the value to be compared for a
-  # match by the translate filter (e.g. `message`, `host`, `response_code`). 
-  # 
-  # If this field is an array, only the first value will be used.
-  config :field, :validate => :string, :required => true
+  # match by the translate filter (e.g. `message`, `host`, `response_code`).
+  #
+  # If this field is an array, only the first value will be used, unless
+  # you specify `iterate_on`. See below. If you want to use another element
+  # in the array then use `"[some_field][2]"`
+  config :source, :validate => :field_reference # effectively :required => true
+  # due compatibility w `field => ...` (non ECS mode) we can not mark it as required
+
+  config :field, :validate => :string, :deprecated => "Use `source` option instead."
 
   # If the destination (or target) field already exists, this configuration item specifies
   # whether the filter should skip translation (default) or overwrite the target field
   # value with the new translation value.
-  config :override, :validate => :boolean, :default => false
+  config :override, :validate => :boolean # :default => false unless field == target
 
   # The dictionary to use for translation, when specified in the logstash filter
   # configuration item (i.e. do not use the `@dictionary_path` file).
@@ -63,8 +81,8 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
 
   # The full path of the external dictionary file. The format of the table should
   # be a standard YAML, JSON or CSV with filenames ending in `.yaml`, `.yml`,
-  #`.json` or `.csv` to be read. Make sure you specify any integer-based keys in 
-  # quotes. For example, the YAML file (`.yaml` or `.yml` should look something like 
+  #`.json` or `.csv` to be read. Make sure you specify any integer-based keys in
+  # quotes. For example, the YAML file (`.yaml` or `.yml` should look something like
   # this:
   # [source,ruby]
   #     "100": Continue
@@ -85,11 +103,13 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # (in seconds) logstash will check the dictionary file for updates.
   config :refresh_interval, :validate => :number, :default => 300
 
-  # The destination field you wish to populate with the translated code. The default
-  # is a field named `translation`. Set this to the same value as source if you want
-  # to do a substitution, in this case filter will allways succeed. This will clobber
-  # the old value of the source field! 
-  config :destination, :validate => :string, :default => "translation"
+  # The target field you wish to populate with the translation.
+  # When ECS Compatibility is enabled, the default is an in-place translation that
+  # will replace the value of the source field.
+  # When ECS Compatibility is disabled, this option falls through to the deprecated `destination` field.
+  config :target, :validate => :field_reference
+
+  config :destination, :validate => :string, :deprecated => "Use `target` option instead." # :default => "translation" (legacy)
 
   # When `exact => true`, the translate filter will populate the destination field
   # with the exact contents of the dictionary value. When `exact => false`, the
@@ -107,11 +127,11 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # will be also set to `bar`. However, if logstash receives an event with the `data` field
   # set to `foofing`, the destination field will be set to `barfing`.
   #
-  # Set both `exact => true` AND `regex => `true` if you would like to match using dictionary
-  # keys as regular expressions. A large dictionary could be expensive to match in this case. 
+  # Set both `exact => true` AND `regex => true` if you would like to match using dictionary
+  # keys as regular expressions. A large dictionary could be expensive to match in this case.
   config :exact, :validate => :boolean, :default => true
 
-  # If you'd like to treat dictionary keys as regular expressions, set `exact => true`.
+  # If you'd like to treat dictionary keys as regular expressions, set `regex => true`.
   # Note: this is activated only when `exact => true`.
   config :regex, :validate => :boolean, :default => false
 
@@ -133,11 +153,21 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
   # deletes old entries on update.
   config :refresh_behaviour, :validate => ['merge', 'replace'], :default => 'merge'
 
+  # When the value that you need to perform enrichment on is a  variable sized array then specify
+  # the field name in this setting. This setting introduces two modes, 1) when the value is an
+  # array of strings and 2) when the value is an array of objects (as in JSON object).
+  # In the first mode, you should have the same field name in both `field` and `iterate_on`, the
+  # result will be an array added to the field specified in the `destination` setting. This array
+  # will have the looked up value (or the `fallback` value or nil) in same ordinal position
+  # as each sought value. In the second mode, specify the field that has the array of objects
+  # then specify the field in each object that provides the sought value with `field` and
+  # the field to write the looked up value (or the `fallback` value) to with `destination`
+  config :iterate_on, :validate => :string
+
+  attr_reader :lookup # for testing reloading
+  attr_reader :updater # for tests
+
   def register
-    rw_lock = java.util.concurrent.locks.ReentrantReadWriteLock.new
-    @read_lock = rw_lock.readLock
-    @write_lock = rw_lock.writeLock
-    
     if @dictionary_path && !@dictionary.empty?
       raise LogStash::ConfigurationError, I18n.t(
         "logstash.agent.configuration.invalid_plugin_register",
@@ -148,153 +178,69 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
     end
 
     if @dictionary_path
-      @next_refresh = Time.now + @refresh_interval
-      raise_exception = true
-      lock_for_write { load_dictionary(raise_exception) }
-    end
-
-    @logger.debug? and @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @dictionary)
-    if @exact
-      @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
+      @lookup = Dictionary::File.create(@dictionary_path, @refresh_interval, @refresh_behaviour, @exact, @regex)
     else
-      @logger.debug? and @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
+      @lookup = Dictionary::Memory.new(@dictionary, @exact, @regex)
     end
-  end # def register
 
-  def lock_for_read
-    @read_lock.lock
-    begin
-      yield
-    ensure
-      @read_lock.unlock
-    end
-  end
-
-  def lock_for_write
-    @write_lock.lock
-    begin
-      yield
-    ensure
-      @write_lock.unlock
-    end
-  end
-
-  def filter(event)
-    if @dictionary_path
-      if needs_refresh?
-        lock_for_write do
-          if needs_refresh?
-            load_dictionary
-            @next_refresh = Time.now + @refresh_interval
-            @logger.info("refreshing dictionary file")
-          end
-        end
+    if @field
+      if @source
+        raise LogStash::ConfigurationError, "Please remove `field => #{@field.inspect}` and only set the `source => ...` option instead"
+      else
+        deprecation_logger.deprecated("`field` option is deprecated; use `source` instead.")
+        logger.debug("intercepting `field` to populate `source`: `#{@field}`")
+        @source = @field
       end
     end
+    unless @source
+      raise LogStash::ConfigurationError, "No source field specified, please provide the `source => ...` option"
+    end
 
-    return unless event.include?(@field) # Skip translation in case event does not have @event field.
-    return if event.include?(@destination) and not @override # Skip translation in case @destination field already exists and @override is disabled.
-
-    begin
-      #If source field is array use first value and make sure source value is string
-      source = event.get(@field).is_a?(Array) ? event.get(@field).first.to_s : event.get(@field).to_s
-      matched = false
-      if @exact
-        if @regex
-          key = @dictionary.keys.detect{|k| source.match(Regexp.new(k))}
-          if key
-            event.set(@destination, lock_for_read { @dictionary[key] })
-            matched = true
-          end
-        elsif @dictionary.include?(source)
-          event.set(@destination, lock_for_read { @dictionary[source] })
-          matched = true
-        end
+    if @destination
+      if @target
+        raise LogStash::ConfigurationError, "Please remove `destination => #{@destination.inspect}` and only set the `target => ...` option instead"
       else
-        translation = lock_for_read { source.gsub(Regexp.union(@dictionary.keys), @dictionary) }
-        
-        if source != translation
-          event.set(@destination, translation.force_encoding(Encoding::UTF_8))
-          matched = true
-        end
+        deprecation_logger.deprecated("`destination` option is deprecated; use `target` instead.")
+        logger.debug("intercepting `destination` to populate `target`: `#{@destination}`")
+        @target = @destination
       end
+    end
+    @target ||= ecs_select[disabled: 'translation', v1: @source]
 
-      if not matched and @fallback
-        event.set(@destination, event.sprintf(@fallback))
-        matched = true
+    if @source == @target
+      @override = true if @override.nil?
+      if @override.eql?(false)
+        raise LogStash::ConfigurationError, "Configuring `override => false` with in-place translation has no effect, please remove the option"
       end
-      filter_matched(event) if matched or @field == @destination
-    rescue Exception => e
-      @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
     end
-  end # def filter
 
-  private
-
-  def load_dictionary(raise_exception=false)
-    if /.y[a]?ml$/.match(@dictionary_path)
-      load_yaml(raise_exception)
-    elsif @dictionary_path.end_with?(".json")
-      load_json(raise_exception)
-    elsif @dictionary_path.end_with?(".csv")
-      load_csv(raise_exception)
+    if @iterate_on.nil?
+      @updater = SingleValueUpdate.new(@source, @target, @fallback, @lookup)
+    elsif @iterate_on == @source
+      @updater = ArrayOfValuesUpdate.new(@iterate_on, @target, @fallback, @lookup)
     else
-      raise "#{self.class.name}: Dictionary #{@dictionary_path} have a non valid format"
-    end
-  rescue => e
-    loading_exception(e, raise_exception)
-  end
-
-  def load_yaml(raise_exception=false)
-    if !File.exists?(@dictionary_path)
-      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
-      return
-    end
-    refresh_dictionary!(YAML.load_file(@dictionary_path))
-  end
-
-  def load_json(raise_exception=false)
-    if !File.exists?(@dictionary_path)
-      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
-      return
-    end
-    refresh_dictionary!(JSON.parse(File.read(@dictionary_path)))
-  end
-
-  def load_csv(raise_exception=false)
-    if !File.exists?(@dictionary_path)
-      @logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
-      return
-    end
-    data = CSV.read(@dictionary_path).inject(Hash.new) do |acc, v|
-      acc[v[0]] = v[1]
-      acc
+      @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @source, @target, @fallback, @lookup)
     end
-    refresh_dictionary!(data)
-  end
 
-  def refresh_dictionary!(data)
-    case @refresh_behaviour
-    when 'merge'
-     @dictionary.merge!(data)
-    when 'replace'
-     @dictionary = data
+    @logger.debug? && @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
+    if @exact
+      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Exact")
     else
-     # we really should never get here
-     raise(LogStash::ConfigurationError, "Unknown value for refresh_behaviour=#{@refresh_behaviour.to_s}")
+      @logger.debug? && @logger.debug("#{self.class.name}: Dictionary translation method - Fuzzy")
     end
-  end
+  end # def register
 
-  def loading_exception(e, raise_exception=false)
-    msg = "#{self.class.name}: #{e.message} when loading dictionary file at #{@dictionary_path}"
-    if raise_exception
-      raise RuntimeError.new(msg)
-    else
-      @logger.warn("#{msg}, continuing with old dictionary", :dictionary_path => @dictionary_path)
-    end
+  def close
+    @lookup.stop_scheduler
   end
 
-  def needs_refresh?
-    @next_refresh < Time.now
-  end
+  def filter(event)
+    return unless @updater.test_for_inclusion(event, @override)
+    begin
+      filter_matched(event) if @updater.update(event) || @source == @target
+    rescue => e
+      @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :source => @source, :event => event.to_hash)
+    end
+  end # def filter
 end # class LogStash::Filters::Translate
+end end