logstash-filter-translate 3.2.2 → 3.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,23 +2,39 @@
2
2
 
3
3
  module LogStash module Filters
4
4
  class SingleValueUpdate
5
+ class CoerceString
6
+ def call(source) source; end
7
+ end
8
+ class CoerceArray
9
+ def call(source) source.first.to_s; end
10
+ end
11
+ class CoerceOther
12
+ def call(source) source.to_s end
13
+ end
14
+
5
15
  def initialize(field, destination, fallback, lookup)
6
16
  @field = field
7
17
  @destination = destination
8
18
  @fallback = fallback
9
19
  @use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
10
20
  @lookup = lookup
21
+ @coercers_table = {}
22
+ @coercers_table.default = CoerceOther.new
23
+ @coercers_table[String] = CoerceString.new
24
+ @coercers_table[Array] = CoerceArray.new
11
25
  end
12
26
 
13
27
  def test_for_inclusion(event, override)
14
28
  # Skip translation in case @destination field already exists and @override is disabled.
15
- return false if event.include?(@destination) && !override
29
+ return false if !override && event.include?(@destination)
16
30
  event.include?(@field)
17
31
  end
18
32
 
19
33
  def update(event)
20
34
  # If source field is array use first value and make sure source value is string
21
- source = Array(event.get(@field)).first.to_s
35
+ # source = Array(event.get(@field)).first.to_s
36
+ source = event.get(@field)
37
+ source = @coercers_table[source.class].call(source)
22
38
  matched = [true, nil]
23
39
  @lookup.fetch_strategy.fetch(source, matched)
24
40
  if matched.first
@@ -1,6 +1,9 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/filters/base"
3
3
  require "logstash/namespace"
4
+ require 'logstash/plugin_mixins/ecs_compatibility_support'
5
+ require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
6
+ require 'logstash/plugin_mixins/deprecation_logger_support'
4
7
 
5
8
  require "logstash/filters/dictionary/memory"
6
9
  require "logstash/filters/dictionary/file"
@@ -35,6 +38,12 @@ require_relative "array_of_maps_value_update"
35
38
  # you might consider using the gsub function of the mutate filter.
36
39
  module LogStash module Filters
37
40
  class Translate < LogStash::Filters::Base
41
+
42
+ include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
43
+ include LogStash::PluginMixins::DeprecationLoggerSupport
44
+
45
+ extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
46
+
38
47
  config_name "translate"
39
48
 
40
49
  # The name of the logstash event field containing the value to be compared for a
@@ -43,12 +52,15 @@ class Translate < LogStash::Filters::Base
43
52
  # If this field is an array, only the first value will be used, unless
44
53
  # you specify `iterate_on`. See below. If you want to use another element
45
54
  # in the array then use `"[some_field][2]"`
46
- config :field, :validate => :string, :required => true
55
+ config :source, :validate => :field_reference # effectively :required => true
56
+ # due compatibility w `field => ...` (non ECS mode) we can not mark it as required
57
+
58
+ config :field, :validate => :string, :deprecated => "Use `source` option instead."
47
59
 
48
60
  # If the destination (or target) field already exists, this configuration item specifies
49
61
  # whether the filter should skip translation (default) or overwrite the target field
50
62
  # value with the new translation value.
51
- config :override, :validate => :boolean, :default => false
63
+ config :override, :validate => :boolean # :default => false unless field == target
52
64
 
53
65
  # The dictionary to use for translation, when specified in the logstash filter
54
66
  # configuration item (i.e. do not use the `@dictionary_path` file).
@@ -91,11 +103,13 @@ class Translate < LogStash::Filters::Base
91
103
  # (in seconds) logstash will check the dictionary file for updates.
92
104
  config :refresh_interval, :validate => :number, :default => 300
93
105
 
94
- # The destination field you wish to populate with the translated code. The default
95
- # is a field named `translation`. Set this to the same value as source if you want
96
- # to do a substitution, in this case filter will allways succeed. This will clobber
97
- # the old value of the source field!
98
- config :destination, :validate => :string, :default => "translation"
106
+ # The target field you wish to populate with the translation.
107
+ # When ECS Compatibility is enabled, the default is an in-place translation that
108
+ # will replace the value of the source field.
109
+ # When ECS Compatibility is disabled, this option falls through to the deprecated `destination` field.
110
+ config :target, :validate => :field_reference
111
+
112
+ config :destination, :validate => :string, :deprecated => "Use `target` option instead." # :default => "translation" (legacy)
99
113
 
100
114
  # When `exact => true`, the translate filter will populate the destination field
101
115
  # with the exact contents of the dictionary value. When `exact => false`, the
@@ -113,7 +127,7 @@ class Translate < LogStash::Filters::Base
113
127
  # will be also set to `bar`. However, if logstash receives an event with the `data` field
114
128
  # set to `foofing`, the destination field will be set to `barfing`.
115
129
  #
116
- # Set both `exact => true` AND `regex => `true` if you would like to match using dictionary
130
+ # Set both `exact => true` AND `regex => true` if you would like to match using dictionary
117
131
  # keys as regular expressions. A large dictionary could be expensive to match in this case.
118
132
  config :exact, :validate => :boolean, :default => true
119
133
 
@@ -151,6 +165,7 @@ class Translate < LogStash::Filters::Base
151
165
  config :iterate_on, :validate => :string
152
166
 
153
167
  attr_reader :lookup # for testing reloading
168
+ attr_reader :updater # for tests
154
169
 
155
170
  def register
156
171
  if @dictionary_path && !@dictionary.empty?
@@ -167,12 +182,44 @@ class Translate < LogStash::Filters::Base
167
182
  else
168
183
  @lookup = Dictionary::Memory.new(@dictionary, @exact, @regex)
169
184
  end
185
+
186
+ if @field
187
+ if @source
188
+ raise LogStash::ConfigurationError, "Please remove `field => #{@field.inspect}` and only set the `source => ...` option instead"
189
+ else
190
+ deprecation_logger.deprecated("`field` option is deprecated; use `source` instead.")
191
+ logger.debug("intercepting `field` to populate `source`: `#{@field}`")
192
+ @source = @field
193
+ end
194
+ end
195
+ unless @source
196
+ raise LogStash::ConfigurationError, "No source field specified, please provide the `source => ...` option"
197
+ end
198
+
199
+ if @destination
200
+ if @target
201
+ raise LogStash::ConfigurationError, "Please remove `destination => #{@destination.inspect}` and only set the `target => ...` option instead"
202
+ else
203
+ deprecation_logger.deprecated("`destination` option is deprecated; use `target` instead.")
204
+ logger.debug("intercepting `destination` to populate `target`: `#{@destination}`")
205
+ @target = @destination
206
+ end
207
+ end
208
+ @target ||= ecs_select[disabled: 'translation', v1: @source]
209
+
210
+ if @source == @target
211
+ @override = true if @override.nil?
212
+ if @override.eql?(false)
213
+ raise LogStash::ConfigurationError, "Configuring `override => false` with in-place translation has no effect, please remove the option"
214
+ end
215
+ end
216
+
170
217
  if @iterate_on.nil?
171
- @updater = SingleValueUpdate.new(@field, @destination, @fallback, @lookup)
172
- elsif @iterate_on == @field
173
- @updater = ArrayOfValuesUpdate.new(@iterate_on, @destination, @fallback, @lookup)
218
+ @updater = SingleValueUpdate.new(@source, @target, @fallback, @lookup)
219
+ elsif @iterate_on == @source
220
+ @updater = ArrayOfValuesUpdate.new(@iterate_on, @target, @fallback, @lookup)
174
221
  else
175
- @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @field, @destination, @fallback, @lookup)
222
+ @updater = ArrayOfMapsValueUpdate.new(@iterate_on, @source, @target, @fallback, @lookup)
176
223
  end
177
224
 
178
225
  @logger.debug? && @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
@@ -190,9 +237,9 @@ class Translate < LogStash::Filters::Base
190
237
  def filter(event)
191
238
  return unless @updater.test_for_inclusion(event, @override)
192
239
  begin
193
- filter_matched(event) if @updater.update(event) || @field == @destination
194
- rescue Exception => e
195
- @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :field => @field, :event => event)
240
+ filter_matched(event) if @updater.update(event) || @source == @target
241
+ rescue => e
242
+ @logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :source => @source, :event => event.to_hash)
196
243
  end
197
244
  end # def filter
198
245
  end # class LogStash::Filters::Translate
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-translate'
4
- s.version = '3.2.2'
4
+ s.version = '3.3.1'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "Replaces field contents based on a hash or YAML file"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,6 +21,9 @@ Gem::Specification.new do |s|
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
24
+ s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.2'
25
+ s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
26
+ s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~> 1.0'
24
27
  s.add_runtime_dependency 'rufus-scheduler'
25
28
 
26
29
  s.add_development_dependency 'logstash-devutils'
@@ -1,4 +1,5 @@
1
1
  # encoding: utf-8
2
+ require 'rspec/wait'
2
3
  require "logstash/devutils/rspec/spec_helper"
3
4
  require "support/rspec_wait_handler_helper"
4
5
  require "support/build_huge_dictionaries"
@@ -14,8 +15,8 @@ describe LogStash::Filters::Translate do
14
15
 
15
16
  let(:config) do
16
17
  {
17
- "field" => "[status]",
18
- "destination" => "[translation]",
18
+ "source" => "[status]",
19
+ "target" => "[translation]",
19
20
  "dictionary_path" => dictionary_path.to_path,
20
21
  "exact" => true,
21
22
  "regex" => false,
@@ -57,7 +58,9 @@ describe LogStash::Filters::Translate do
57
58
  end
58
59
  .then_after(1.2, "wait then translate again") do
59
60
  subject.filter(event)
60
- wait(0.1).for{event.get("[translation]")}.to eq("12"), "field [translation] did not eq '12'"
61
+ try(5) do
62
+ wait(0.1).for{event.get("[translation]")}.to eq("12"), "field [translation] did not eq '12'"
63
+ end
61
64
  end
62
65
  .then("stop") do
63
66
  subject.close
@@ -86,7 +89,9 @@ describe LogStash::Filters::Translate do
86
89
  end
87
90
  .then_after(1.2, "wait then translate again") do
88
91
  subject.filter(event)
89
- wait(0.1).for{event.get("[translation]")}.to eq("22"), "field [translation] did not eq '22'"
92
+ try(5) do
93
+ wait(0.1).for{event.get("[translation]")}.to eq("22"), "field [translation] did not eq '22'"
94
+ end
90
95
  end
91
96
  .then("stop") do
92
97
  subject.close
@@ -105,8 +110,8 @@ describe LogStash::Filters::Translate do
105
110
  let(:dictionary_size) { 100000 }
106
111
  let(:config) do
107
112
  {
108
- "field" => "[status]",
109
- "destination" => "[translation]",
113
+ "source" => "[status]",
114
+ "target" => "[translation]",
110
115
  "dictionary_path" => dictionary_path.to_path,
111
116
  "exact" => true,
112
117
  "regex" => false,
@@ -154,8 +159,8 @@ describe LogStash::Filters::Translate do
154
159
  let(:dictionary_size) { 100000 }
155
160
  let(:config) do
156
161
  {
157
- "field" => "[status]",
158
- "destination" => "[translation]",
162
+ "source" => "[status]",
163
+ "target" => "[translation]",
159
164
  "dictionary_path" => dictionary_path.to_path,
160
165
  "exact" => true,
161
166
  "regex" => false,