logstash-filter-translate 3.2.2 → 3.3.1

data/lib/logstash/filters/single_value_update.rb

@@ -2,23 +2,39 @@
 
 module LogStash module Filters
   class SingleValueUpdate
+    class CoerceString
+      def call(source) source; end
+    end
+    class CoerceArray
+      def call(source) source.first.to_s; end
+    end
+    class CoerceOther
+      def call(source) source.to_s end
+    end
+
     def initialize(field, destination, fallback, lookup)
       @field = field
       @destination = destination
       @fallback = fallback
       @use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
       @lookup = lookup
+      @coercers_table = {}
+      @coercers_table.default = CoerceOther.new
+      @coercers_table[String] = CoerceString.new
+      @coercers_table[Array] = CoerceArray.new
     end
 
     def test_for_inclusion(event, override)
       # Skip translation in case @destination field already exists and @override is disabled.
-      return false if event.include?(@destination) && !override
+      return false if !override && event.include?(@destination)
       event.include?(@field)
     end
 
     def update(event)
       # If source field is array use first value and make sure source value is string
-      source = Array(event.get(@field)).first.to_s
+      # source = Array(event.get(@field)).first.to_s
+      source = event.get(@field)
+      source = @coercers_table[source.class].call(source)
       matched = [true, nil]
       @lookup.fetch_strategy.fetch(source, matched)
       if matched.first

data/lib/logstash/filters/translate.rb

@@ -1,6 +1,9 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
+require 'logstash/plugin_mixins/deprecation_logger_support'
 
 require "logstash/filters/dictionary/memory"
 require "logstash/filters/dictionary/file"
@@ -35,6 +38,12 @@ require_relative "array_of_maps_value_update"
 # you might consider using the gsub function of the mutate filter.
 module LogStash module Filters
 class Translate < LogStash::Filters::Base
+
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::DeprecationLoggerSupport
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   config_name "translate"
 
   # The name of the logstash event field containing the value to be compared for a
@@ -43,12 +52,15 @@ class Translate < LogStash::Filters::Base
   # If this field is an array, only the first value will be used, unless
   # you specify `iterate_on`. See below. If you want to use another element
   # in the array then use `"[some_field][2]"`
-  config :field, :validate => :string, :required => true
+  config :source, :validate => :field_reference # effectively :required => true
+  # due compatibility w `field => ...` (non ECS mode) we can not mark it as required
+
+  config :field, :validate => :string, :deprecated => "Use `source` option instead."
 
   # If the destination (or target) field already exists, this configuration item specifies
   # whether the filter should skip translation (default) or overwrite the target field
   # value with the new translation value.
-  config :override, :validate => :boolean, :default => false
+  config :override, :validate => :boolean # :default => false unless field == target
 
   # The dictionary to use for translation, when specified in the logstash filter
   # configuration item (i.e. do not use the `@dictionary_path` file).
@@ -91,11 +103,13 @@ class Translate < LogStash::Filters::Base
   # (in seconds) logstash will check the dictionary file for updates.
   config :refresh_interval, :validate => :number, :default => 300
 
-  # The destination field you wish to populate with the translated code. The default
-  # is a field named `translation`. Set this to the same value as source if you want
-  # to do a substitution, in this case filter will allways succeed. This will clobber
-  # the old value of the source field!
-  config :destination, :validate => :string, :default => "translation"
+  # The target field you wish to populate with the translation.
+  # When ECS Compatibility is enabled, the default is an in-place translation that
+  # will replace the value of the source field.
+  # When ECS Compatibility is disabled, this option falls through to the deprecated `destination` field.
+  config :target, :validate => :field_reference
+
+  config :destination, :validate => :string, :deprecated => "Use `target` option instead." # :default => "translation" (legacy)
 
   # When `exact => true`, the translate filter will populate the destination field
   # with the exact contents of the dictionary value. When `exact => false`, the
@@ -113,7 +127,7 @@ class Translate < LogStash::Filters::Base
   # will be also set to `bar`. However, if logstash receives an event with the `data` field
   # set to `foofing`, the destination field will be set to `barfing`.
   #
-  # Set both `exact => true` AND `regex => `true` if you would like to match using dictionary
+  # Set both `exact => true` AND `regex => true` if you would like to match using dictionary
   # keys as regular expressions. A large dictionary could be expensive to match in this case.
   config :exact, :validate => :boolean, :default => true
 
@@ -151,6 +165,7 @@ class Translate < LogStash::Filters::Base
   config :iterate_on, :validate => :string
 
   attr_reader :lookup # for testing reloading
+  attr_reader :updater # for tests
 
   def register
     if @dictionary_path && !@dictionary.empty?
@@ -167,12 +182,44 @@ class Translate < LogStash::Filters::Base
     else
       @lookup = Dictionary::Memory.new(@dictionary, @exact, @regex)
     end
+
+    if @field
+      if @source
+        raise LogStash::ConfigurationError, "Please remove `field => #{@field.inspect}` and only set the `source => ...` option instead"
+      else
+        deprecation_logger.deprecated("`field` option is deprecated; use `source` instead.")
+        logger.debug("intercepting `field` to populate `source`: `#{@field}`")
+        @source = @field
+      end
+    end
+    unless @source
+      raise LogStash::ConfigurationError, "No source field specified, please provide the `source => ...` option"
+    end
+
+    if @destination
+      if @target
+        raise LogStash::ConfigurationError, "Please remove `destination => #{@destination.inspect}` and only set the `target => ...` option instead"
+      else
+        deprecation_logger.deprecated("`destination` option is deprecated; use `target` instead.")
+        logger.debug("intercepting `destination` to populate `target`: `#{@destination}`")
+        @target = @destination
+      end
+    end
+    @target ||= ecs_select[disabled: 'translation', v1: @source]
+
+    if @source == @target
+      @override = true if @override.nil?
+      if @override.eql?(false)
+        raise LogStash::ConfigurationError, "Configuring `override => false` with in-place translation has no effect, please remove the option"
+      end
+    end
+
     if @iterate_on.nil?
-      @updater = SingleValueUpdate.new(@field, @destination, @fallback, @lookup)
-    elsif @iterate_on == @field
-      @updater = ArrayOfValuesUpdate.new(@iterate_on, @destination, @fallback, @lookup)
+      @updater = SingleValueUpdate.new(@source, @target, @fallback, @lookup)
+    elsif @iterate_on == @source
+      @updater = ArrayOfValuesUpdate.new(@iterate_on, @target, @fallback, @lookup)
     else
-      @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @field, @destination, @fallback, @lookup)
+      @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @source, @target, @fallback, @lookup)
     end
 
     @logger.debug? && @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
@@ -190,9 +237,9 @@ class Translate < LogStash::Filters::Base
   def filter(event)
     return unless @updater.test_for_inclusion(event, @override)
     begin
-      filter_matched(event) if @updater.update(event) || @field == @destination
-    rescue Exception => e
-      @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
+      filter_matched(event) if @updater.update(event) || @source == @target
+    rescue => e
+      @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :source => @source, :event => event.to_hash)
     end
   end # def filter
 end # class LogStash::Filters::Translate

data/logstash-filter-translate.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-translate'
-  s.version         = '3.2.2'
+  s.version         = '3.3.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Replaces field contents based on a hash or YAML file"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,6 +21,9 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.2'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~> 1.0'
   s.add_runtime_dependency 'rufus-scheduler'
 
   s.add_development_dependency 'logstash-devutils'

data/spec/filters/scheduling_spec.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require 'rspec/wait'
 require "logstash/devutils/rspec/spec_helper"
 require "support/rspec_wait_handler_helper"
 require "support/build_huge_dictionaries"
@@ -14,8 +15,8 @@ describe LogStash::Filters::Translate do
 
     let(:config) do
       {
-        "field"       => "[status]",
-        "destination" => "[translation]",
+        "source"      => "[status]",
+        "target"      => "[translation]",
         "dictionary_path"  => dictionary_path.to_path,
         "exact"       => true,
         "regex"       => false,
@@ -57,7 +58,9 @@ describe LogStash::Filters::Translate do
           end
           .then_after(1.2, "wait then translate again") do
             subject.filter(event)
-            wait(0.1).for{event.get("[translation]")}.to eq("12"), "field [translation] did not eq '12'"
+            try(5) do
+              wait(0.1).for{event.get("[translation]")}.to eq("12"), "field [translation] did not eq '12'"
+            end
           end
           .then("stop") do
             subject.close
@@ -86,7 +89,9 @@ describe LogStash::Filters::Translate do
           end
           .then_after(1.2, "wait then translate again") do
             subject.filter(event)
-            wait(0.1).for{event.get("[translation]")}.to eq("22"), "field [translation] did not eq '22'"
+            try(5) do
+              wait(0.1).for{event.get("[translation]")}.to eq("22"), "field [translation] did not eq '22'"
+            end
           end
           .then("stop") do
             subject.close
@@ -105,8 +110,8 @@ describe LogStash::Filters::Translate do
     let(:dictionary_size) { 100000 }
     let(:config) do
       {
-        "field"       => "[status]",
-        "destination" => "[translation]",
+        "source"      => "[status]",
+        "target"      => "[translation]",
         "dictionary_path"  => dictionary_path.to_path,
         "exact"       => true,
         "regex"       => false,
@@ -154,8 +159,8 @@ describe LogStash::Filters::Translate do
     let(:dictionary_size) { 100000 }
     let(:config) do
       {
-        "field"       => "[status]",
-        "destination" => "[translation]",
+        "source"      => "[status]",
+        "target"      => "[translation]",
         "dictionary_path"  => dictionary_path.to_path,
         "exact"       => true,
         "regex"       => false,