logstash-filter-translate 3.2.2 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/LICENSE +199 -10
- data/README.md +1 -1
- data/docs/index.asciidoc +92 -42
- data/lib/logstash/filters/array_of_maps_value_update.rb +1 -1
- data/lib/logstash/filters/array_of_values_update.rb +13 -3
- data/lib/logstash/filters/dictionary/csv_file.rb +2 -12
- data/lib/logstash/filters/dictionary/file.rb +9 -6
- data/lib/logstash/filters/dictionary/memory.rb +6 -5
- data/lib/logstash/filters/single_value_update.rb +18 -2
- data/lib/logstash/filters/translate.rb +62 -15
- data/logstash-filter-translate.gemspec +4 -1
- data/spec/filters/scheduling_spec.rb +13 -8
- data/spec/filters/translate_spec.rb +285 -88
- metadata +45 -4
@@ -2,23 +2,39 @@
|
|
2
2
|
|
3
3
|
module LogStash module Filters
|
4
4
|
class SingleValueUpdate
|
5
|
+
class CoerceString
|
6
|
+
def call(source) source; end
|
7
|
+
end
|
8
|
+
class CoerceArray
|
9
|
+
def call(source) source.first.to_s; end
|
10
|
+
end
|
11
|
+
class CoerceOther
|
12
|
+
def call(source) source.to_s end
|
13
|
+
end
|
14
|
+
|
5
15
|
def initialize(field, destination, fallback, lookup)
|
6
16
|
@field = field
|
7
17
|
@destination = destination
|
8
18
|
@fallback = fallback
|
9
19
|
@use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
|
10
20
|
@lookup = lookup
|
21
|
+
@coercers_table = {}
|
22
|
+
@coercers_table.default = CoerceOther.new
|
23
|
+
@coercers_table[String] = CoerceString.new
|
24
|
+
@coercers_table[Array] = CoerceArray.new
|
11
25
|
end
|
12
26
|
|
13
27
|
def test_for_inclusion(event, override)
|
14
28
|
# Skip translation in case @destination field already exists and @override is disabled.
|
15
|
-
return false if event.include?(@destination)
|
29
|
+
return false if !override && event.include?(@destination)
|
16
30
|
event.include?(@field)
|
17
31
|
end
|
18
32
|
|
19
33
|
def update(event)
|
20
34
|
# If source field is array use first value and make sure source value is string
|
21
|
-
source = Array(event.get(@field)).first.to_s
|
35
|
+
# source = Array(event.get(@field)).first.to_s
|
36
|
+
source = event.get(@field)
|
37
|
+
source = @coercers_table[source.class].call(source)
|
22
38
|
matched = [true, nil]
|
23
39
|
@lookup.fetch_strategy.fetch(source, matched)
|
24
40
|
if matched.first
|
@@ -1,6 +1,9 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/filters/base"
|
3
3
|
require "logstash/namespace"
|
4
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
5
|
+
require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
|
6
|
+
require 'logstash/plugin_mixins/deprecation_logger_support'
|
4
7
|
|
5
8
|
require "logstash/filters/dictionary/memory"
|
6
9
|
require "logstash/filters/dictionary/file"
|
@@ -35,6 +38,12 @@ require_relative "array_of_maps_value_update"
|
|
35
38
|
# you might consider using the gsub function of the mutate filter.
|
36
39
|
module LogStash module Filters
|
37
40
|
class Translate < LogStash::Filters::Base
|
41
|
+
|
42
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
43
|
+
include LogStash::PluginMixins::DeprecationLoggerSupport
|
44
|
+
|
45
|
+
extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
|
46
|
+
|
38
47
|
config_name "translate"
|
39
48
|
|
40
49
|
# The name of the logstash event field containing the value to be compared for a
|
@@ -43,12 +52,15 @@ class Translate < LogStash::Filters::Base
|
|
43
52
|
# If this field is an array, only the first value will be used, unless
|
44
53
|
# you specify `iterate_on`. See below. If you want to use another element
|
45
54
|
# in the array then use `"[some_field][2]"`
|
46
|
-
config :
|
55
|
+
config :source, :validate => :field_reference # effectively :required => true
|
56
|
+
# due compatibility w `field => ...` (non ECS mode) we can not mark it as required
|
57
|
+
|
58
|
+
config :field, :validate => :string, :deprecated => "Use `source` option instead."
|
47
59
|
|
48
60
|
# If the destination (or target) field already exists, this configuration item specifies
|
49
61
|
# whether the filter should skip translation (default) or overwrite the target field
|
50
62
|
# value with the new translation value.
|
51
|
-
config :override, :validate => :boolean
|
63
|
+
config :override, :validate => :boolean # :default => false unless field == target
|
52
64
|
|
53
65
|
# The dictionary to use for translation, when specified in the logstash filter
|
54
66
|
# configuration item (i.e. do not use the `@dictionary_path` file).
|
@@ -91,11 +103,13 @@ class Translate < LogStash::Filters::Base
|
|
91
103
|
# (in seconds) logstash will check the dictionary file for updates.
|
92
104
|
config :refresh_interval, :validate => :number, :default => 300
|
93
105
|
|
94
|
-
# The
|
95
|
-
#
|
96
|
-
#
|
97
|
-
#
|
98
|
-
config :
|
106
|
+
# The target field you wish to populate with the translation.
|
107
|
+
# When ECS Compatibility is enabled, the default is an in-place translation that
|
108
|
+
# will replace the value of the source field.
|
109
|
+
# When ECS Compatibility is disabled, this option falls through to the deprecated `destination` field.
|
110
|
+
config :target, :validate => :field_reference
|
111
|
+
|
112
|
+
config :destination, :validate => :string, :deprecated => "Use `target` option instead." # :default => "translation" (legacy)
|
99
113
|
|
100
114
|
# When `exact => true`, the translate filter will populate the destination field
|
101
115
|
# with the exact contents of the dictionary value. When `exact => false`, the
|
@@ -113,7 +127,7 @@ class Translate < LogStash::Filters::Base
|
|
113
127
|
# will be also set to `bar`. However, if logstash receives an event with the `data` field
|
114
128
|
# set to `foofing`, the destination field will be set to `barfing`.
|
115
129
|
#
|
116
|
-
# Set both `exact => true` AND `regex =>
|
130
|
+
# Set both `exact => true` AND `regex => true` if you would like to match using dictionary
|
117
131
|
# keys as regular expressions. A large dictionary could be expensive to match in this case.
|
118
132
|
config :exact, :validate => :boolean, :default => true
|
119
133
|
|
@@ -151,6 +165,7 @@ class Translate < LogStash::Filters::Base
|
|
151
165
|
config :iterate_on, :validate => :string
|
152
166
|
|
153
167
|
attr_reader :lookup # for testing reloading
|
168
|
+
attr_reader :updater # for tests
|
154
169
|
|
155
170
|
def register
|
156
171
|
if @dictionary_path && !@dictionary.empty?
|
@@ -167,12 +182,44 @@ class Translate < LogStash::Filters::Base
|
|
167
182
|
else
|
168
183
|
@lookup = Dictionary::Memory.new(@dictionary, @exact, @regex)
|
169
184
|
end
|
185
|
+
|
186
|
+
if @field
|
187
|
+
if @source
|
188
|
+
raise LogStash::ConfigurationError, "Please remove `field => #{@field.inspect}` and only set the `source => ...` option instead"
|
189
|
+
else
|
190
|
+
deprecation_logger.deprecated("`field` option is deprecated; use `source` instead.")
|
191
|
+
logger.debug("intercepting `field` to populate `source`: `#{@field}`")
|
192
|
+
@source = @field
|
193
|
+
end
|
194
|
+
end
|
195
|
+
unless @source
|
196
|
+
raise LogStash::ConfigurationError, "No source field specified, please provide the `source => ...` option"
|
197
|
+
end
|
198
|
+
|
199
|
+
if @destination
|
200
|
+
if @target
|
201
|
+
raise LogStash::ConfigurationError, "Please remove `destination => #{@destination.inspect}` and only set the `target => ...` option instead"
|
202
|
+
else
|
203
|
+
deprecation_logger.deprecated("`destination` option is deprecated; use `target` instead.")
|
204
|
+
logger.debug("intercepting `destination` to populate `target`: `#{@destination}`")
|
205
|
+
@target = @destination
|
206
|
+
end
|
207
|
+
end
|
208
|
+
@target ||= ecs_select[disabled: 'translation', v1: @source]
|
209
|
+
|
210
|
+
if @source == @target
|
211
|
+
@override = true if @override.nil?
|
212
|
+
if @override.eql?(false)
|
213
|
+
raise LogStash::ConfigurationError, "Configuring `override => false` with in-place translation has no effect, please remove the option"
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
170
217
|
if @iterate_on.nil?
|
171
|
-
@updater = SingleValueUpdate.new(@
|
172
|
-
elsif @iterate_on == @
|
173
|
-
@updater = ArrayOfValuesUpdate.new(@iterate_on, @
|
218
|
+
@updater = SingleValueUpdate.new(@source, @target, @fallback, @lookup)
|
219
|
+
elsif @iterate_on == @source
|
220
|
+
@updater = ArrayOfValuesUpdate.new(@iterate_on, @target, @fallback, @lookup)
|
174
221
|
else
|
175
|
-
@updater = ArrayOfMapsValueUpdate.new(@iterate_on, @
|
222
|
+
@updater = ArrayOfMapsValueUpdate.new(@iterate_on, @source, @target, @fallback, @lookup)
|
176
223
|
end
|
177
224
|
|
178
225
|
@logger.debug? && @logger.debug("#{self.class.name}: Dictionary - ", :dictionary => @lookup.dictionary)
|
@@ -190,9 +237,9 @@ class Translate < LogStash::Filters::Base
|
|
190
237
|
def filter(event)
|
191
238
|
return unless @updater.test_for_inclusion(event, @override)
|
192
239
|
begin
|
193
|
-
filter_matched(event) if @updater.update(event) || @
|
194
|
-
rescue
|
195
|
-
@logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :
|
240
|
+
filter_matched(event) if @updater.update(event) || @source == @target
|
241
|
+
rescue => e
|
242
|
+
@logger.error("Something went wrong when attempting to translate from dictionary", :exception => e, :source => @source, :event => event.to_hash)
|
196
243
|
end
|
197
244
|
end # def filter
|
198
245
|
end # class LogStash::Filters::Translate
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-translate'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.3.1'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Replaces field contents based on a hash or YAML file"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -21,6 +21,9 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
+
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.2'
|
25
|
+
s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
|
26
|
+
s.add_runtime_dependency 'logstash-mixin-deprecation_logger_support', '~> 1.0'
|
24
27
|
s.add_runtime_dependency 'rufus-scheduler'
|
25
28
|
|
26
29
|
s.add_development_dependency 'logstash-devutils'
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
+
require 'rspec/wait'
|
2
3
|
require "logstash/devutils/rspec/spec_helper"
|
3
4
|
require "support/rspec_wait_handler_helper"
|
4
5
|
require "support/build_huge_dictionaries"
|
@@ -14,8 +15,8 @@ describe LogStash::Filters::Translate do
|
|
14
15
|
|
15
16
|
let(:config) do
|
16
17
|
{
|
17
|
-
"
|
18
|
-
"
|
18
|
+
"source" => "[status]",
|
19
|
+
"target" => "[translation]",
|
19
20
|
"dictionary_path" => dictionary_path.to_path,
|
20
21
|
"exact" => true,
|
21
22
|
"regex" => false,
|
@@ -57,7 +58,9 @@ describe LogStash::Filters::Translate do
|
|
57
58
|
end
|
58
59
|
.then_after(1.2, "wait then translate again") do
|
59
60
|
subject.filter(event)
|
60
|
-
|
61
|
+
try(5) do
|
62
|
+
wait(0.1).for{event.get("[translation]")}.to eq("12"), "field [translation] did not eq '12'"
|
63
|
+
end
|
61
64
|
end
|
62
65
|
.then("stop") do
|
63
66
|
subject.close
|
@@ -86,7 +89,9 @@ describe LogStash::Filters::Translate do
|
|
86
89
|
end
|
87
90
|
.then_after(1.2, "wait then translate again") do
|
88
91
|
subject.filter(event)
|
89
|
-
|
92
|
+
try(5) do
|
93
|
+
wait(0.1).for{event.get("[translation]")}.to eq("22"), "field [translation] did not eq '22'"
|
94
|
+
end
|
90
95
|
end
|
91
96
|
.then("stop") do
|
92
97
|
subject.close
|
@@ -105,8 +110,8 @@ describe LogStash::Filters::Translate do
|
|
105
110
|
let(:dictionary_size) { 100000 }
|
106
111
|
let(:config) do
|
107
112
|
{
|
108
|
-
"
|
109
|
-
"
|
113
|
+
"source" => "[status]",
|
114
|
+
"target" => "[translation]",
|
110
115
|
"dictionary_path" => dictionary_path.to_path,
|
111
116
|
"exact" => true,
|
112
117
|
"regex" => false,
|
@@ -154,8 +159,8 @@ describe LogStash::Filters::Translate do
|
|
154
159
|
let(:dictionary_size) { 100000 }
|
155
160
|
let(:config) do
|
156
161
|
{
|
157
|
-
"
|
158
|
-
"
|
162
|
+
"source" => "[status]",
|
163
|
+
"target" => "[translation]",
|
159
164
|
"dictionary_path" => dictionary_path.to_path,
|
160
165
|
"exact" => true,
|
161
166
|
"regex" => false,
|