logstash-filter-translate 3.2.2 → 3.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/docs/index.asciidoc +20 -10
- data/lib/logstash/filters/array_of_maps_value_update.rb +1 -1
- data/lib/logstash/filters/array_of_values_update.rb +13 -3
- data/lib/logstash/filters/dictionary/file.rb +9 -6
- data/lib/logstash/filters/dictionary/memory.rb +6 -5
- data/lib/logstash/filters/single_value_update.rb +18 -2
- data/logstash-filter-translate.gemspec +1 -1
- data/spec/filters/translate_spec.rb +79 -37
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 21079c87f27ea96b5aafd79d5a48405a22fdbf1e7ad14de01a4f5f79b064b21f
|
4
|
+
data.tar.gz: b0c6f572eaf3d2dc53887838d1947ab5a8aac11a8718dc9a1f5f4fbe695fcc48
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 53c86c7099d90e31007f68f83e29f5867d1f81e162b0fe01fdde8ca07d3e3a063250586fff399193cee50366b360da60602711c38515ca4b7722eb36edcf6c1d
|
7
|
+
data.tar.gz: 1d4db2634a1d862bce62fdd3dd150b5985a657cdc78d7f2b90426dd5765879aa1a654ad30aec7f43bf93abd3edd220333002dcd03b42a971a77835a8a4faed59
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 3.2.3
|
2
|
+
- Fix to align with docs - looked-up values are always strings. Coerce better. [#77](https://github.com/logstash-plugins/logstash-filter-translate/pull/77)
|
3
|
+
- Fix bug in dictionary/file the always applied RegexExact, manifested when dictionary keys are not regex compatible [Logstash #9936](https://github.com/elastic/logstash/issues/9936)
|
4
|
+
- Added info to dictionary_path description to explain why integers must be quoted
|
5
|
+
|
1
6
|
## 3.2.2
|
2
7
|
- Fix bug in csv_file when LS config has CSV filter plugin specified as well as a csv dictionary.
|
3
8
|
[#70](https://github.com/logstash-plugins/logstash-filter-translate/issues/70)
|
data/docs/index.asciidoc
CHANGED
@@ -24,11 +24,14 @@ A general search and replace tool that uses a configured hash
|
|
24
24
|
and/or a file to determine replacement values. Currently supported are
|
25
25
|
YAML, JSON, and CSV files. Each dictionary item is a key value pair.
|
26
26
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
27
|
+
You can specify dictionary entries in one of two ways:
|
28
|
+
|
29
|
+
* The `dictionary` configuration item can contain a hash representing
|
30
|
+
the mapping.
|
31
|
+
* An external file (readable by logstash) may be specified in the
|
32
|
+
`dictionary_path` configuration item.
|
33
|
+
|
34
|
+
These two methods may not be used in conjunction; it will produce an error.
|
32
35
|
|
33
36
|
Operationally, for each event, the value from the `field` setting is tested
|
34
37
|
against the dictionary and if it matches exactly (or matches a regex when
|
@@ -146,9 +149,14 @@ NOTE: It is an error to specify both `dictionary` and `dictionary_path`.
|
|
146
149
|
* Value type is <<path,path>>
|
147
150
|
* There is no default value for this setting.
|
148
151
|
|
149
|
-
The full path of the external dictionary file. The format of the table
|
150
|
-
|
151
|
-
|
152
|
+
The full path of the external dictionary file. The format of the table should be
|
153
|
+
a standard YAML, JSON, or CSV.
|
154
|
+
|
155
|
+
Specify any integer-based keys in quotes. The
|
156
|
+
value taken from the event's `field` setting is converted to a string. The
|
157
|
+
lookup dictionary keys must also be strings, and the quotes make the
|
158
|
+
integer-based keys function as a string. For example, the YAML file should look
|
159
|
+
something like this:
|
152
160
|
|
153
161
|
[source,ruby]
|
154
162
|
----
|
@@ -354,8 +362,10 @@ A value of zero or less will disable refresh.
|
|
354
362
|
* Value type is <<boolean,boolean>>
|
355
363
|
* Default value is `false`
|
356
364
|
|
357
|
-
|
358
|
-
|
365
|
+
To treat dictionary keys as regular expressions, set `regex => true`.
|
366
|
+
|
367
|
+
Be sure to escape dictionary key strings for use with regex.
|
368
|
+
Resources on regex formatting are available online.
|
359
369
|
|
360
370
|
[id="plugins-{type}s-{plugin}-refresh_behaviour"]
|
361
371
|
===== `refresh_behaviour`
|
@@ -25,7 +25,7 @@ module LogStash module Filters
|
|
25
25
|
inner = event.get(nested_field)
|
26
26
|
next if inner.nil?
|
27
27
|
matched = [true, nil]
|
28
|
-
@lookup.fetch_strategy.fetch(inner, matched)
|
28
|
+
@lookup.fetch_strategy.fetch(inner.to_s, matched)
|
29
29
|
if matched.first
|
30
30
|
event.set(nested_destination, matched.last)
|
31
31
|
matches[index] = true
|
@@ -2,30 +2,40 @@
|
|
2
2
|
|
3
3
|
module LogStash module Filters
|
4
4
|
class ArrayOfValuesUpdate
|
5
|
+
class CoerceArray
|
6
|
+
def call(source) source; end
|
7
|
+
end
|
8
|
+
class CoerceOther
|
9
|
+
def call(source) Array(source); end
|
10
|
+
end
|
11
|
+
|
5
12
|
def initialize(iterate_on, destination, fallback, lookup)
|
6
13
|
@iterate_on = iterate_on
|
7
14
|
@destination = destination
|
8
15
|
@fallback = fallback
|
9
16
|
@use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
|
10
17
|
@lookup = lookup
|
18
|
+
@coercers_table = {}
|
19
|
+
@coercers_table.default = CoerceOther.new
|
20
|
+
@coercers_table[Array] = CoerceArray.new
|
11
21
|
end
|
12
22
|
|
13
23
|
def test_for_inclusion(event, override)
|
14
24
|
# Skip translation in case @destination iterate_on already exists and @override is disabled.
|
15
|
-
return false if event.include?(@destination)
|
25
|
+
return false if !override && event.include?(@destination)
|
16
26
|
event.include?(@iterate_on)
|
17
27
|
end
|
18
28
|
|
19
29
|
def update(event)
|
20
30
|
val = event.get(@iterate_on)
|
21
|
-
source =
|
31
|
+
source = @coercers_table[val.class].call(val)
|
22
32
|
target = Array.new(source.size)
|
23
33
|
if @use_fallback
|
24
34
|
target.fill(event.sprintf(@fallback))
|
25
35
|
end
|
26
36
|
source.each_with_index do |inner, index|
|
27
37
|
matched = [true, nil]
|
28
|
-
@lookup.fetch_strategy.fetch(inner, matched)
|
38
|
+
@lookup.fetch_strategy.fetch(inner.to_s, matched)
|
29
39
|
if matched.first
|
30
40
|
target[index] = matched.last
|
31
41
|
end
|
@@ -44,11 +44,12 @@ module LogStash module Filters module Dictionary
|
|
44
44
|
@update_method = method(:merge_dictionary)
|
45
45
|
initialize_for_file_type
|
46
46
|
args = [@dictionary, rw_lock]
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
47
|
+
klass = case
|
48
|
+
when exact && regex then FetchStrategy::File::ExactRegex
|
49
|
+
when exact then FetchStrategy::File::Exact
|
50
|
+
else FetchStrategy::File::RegexUnion
|
51
|
+
end
|
52
|
+
@fetch_strategy = klass.new(*args)
|
52
53
|
load_dictionary(raise_exception = true)
|
53
54
|
stop_scheduler(initial = true)
|
54
55
|
start_scheduler unless @refresh_interval <= 0 # disabled, a scheduler interval of zero makes no sense
|
@@ -131,7 +132,9 @@ module LogStash module Filters module Dictionary
|
|
131
132
|
def loading_exception(e, raise_exception)
|
132
133
|
msg = "Translate: #{e.message} when loading dictionary file at #{@dictionary_path}"
|
133
134
|
if raise_exception
|
134
|
-
|
135
|
+
dfe = DictionaryFileError.new(msg)
|
136
|
+
dfe.set_backtrace(e.backtrace)
|
137
|
+
raise dfe
|
135
138
|
else
|
136
139
|
@logger.warn("#{msg}, continuing with old dictionary", :dictionary_path => @dictionary_path)
|
137
140
|
end
|
@@ -7,11 +7,12 @@ module LogStash module Filters module Dictionary
|
|
7
7
|
attr_reader :dictionary, :fetch_strategy
|
8
8
|
|
9
9
|
def initialize(hash, exact, regex)
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
10
|
+
klass = case
|
11
|
+
when exact && regex then FetchStrategy::Memory::ExactRegex
|
12
|
+
when exact then FetchStrategy::Memory::Exact
|
13
|
+
else FetchStrategy::Memory::RegexUnion
|
14
|
+
end
|
15
|
+
@fetch_strategy = klass.new(hash)
|
15
16
|
end
|
16
17
|
|
17
18
|
def stop_scheduler
|
@@ -2,23 +2,39 @@
|
|
2
2
|
|
3
3
|
module LogStash module Filters
|
4
4
|
class SingleValueUpdate
|
5
|
+
class CoerceString
|
6
|
+
def call(source) source; end
|
7
|
+
end
|
8
|
+
class CoerceArray
|
9
|
+
def call(source) source.first.to_s; end
|
10
|
+
end
|
11
|
+
class CoerceOther
|
12
|
+
def call(source) source.to_s end
|
13
|
+
end
|
14
|
+
|
5
15
|
def initialize(field, destination, fallback, lookup)
|
6
16
|
@field = field
|
7
17
|
@destination = destination
|
8
18
|
@fallback = fallback
|
9
19
|
@use_fallback = !fallback.nil? # fallback is not nil, the user set a value in the config
|
10
20
|
@lookup = lookup
|
21
|
+
@coercers_table = {}
|
22
|
+
@coercers_table.default = CoerceOther.new
|
23
|
+
@coercers_table[String] = CoerceString.new
|
24
|
+
@coercers_table[Array] = CoerceArray.new
|
11
25
|
end
|
12
26
|
|
13
27
|
def test_for_inclusion(event, override)
|
14
28
|
# Skip translation in case @destination field already exists and @override is disabled.
|
15
|
-
return false if event.include?(@destination)
|
29
|
+
return false if !override && event.include?(@destination)
|
16
30
|
event.include?(@field)
|
17
31
|
end
|
18
32
|
|
19
33
|
def update(event)
|
20
34
|
# If source field is array use first value and make sure source value is string
|
21
|
-
source = Array(event.get(@field)).first.to_s
|
35
|
+
# source = Array(event.get(@field)).first.to_s
|
36
|
+
source = event.get(@field)
|
37
|
+
source = @coercers_table[source.class].call(source)
|
22
38
|
matched = [true, nil]
|
23
39
|
@lookup.fetch_strategy.fetch(source, matched)
|
24
40
|
if matched.first
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-translate'
|
4
|
-
s.version = '3.2.
|
4
|
+
s.version = '3.2.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Replaces field contents based on a hash or YAML file"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -2,6 +2,12 @@
|
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/filters/translate"
|
4
4
|
|
5
|
+
module TranslateUtil
|
6
|
+
def self.build_fixture_path(filename)
|
7
|
+
File.join(File.dirname(__FILE__), "..", "fixtures", filename)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
5
11
|
describe LogStash::Filters::Translate do
|
6
12
|
|
7
13
|
let(:config) { Hash.new }
|
@@ -24,13 +30,36 @@ describe LogStash::Filters::Translate do
|
|
24
30
|
|
25
31
|
let(:event) { LogStash::Event.new("status" => 200) }
|
26
32
|
|
27
|
-
it "
|
33
|
+
it "coerces field to a string then returns the exact translation" do
|
28
34
|
subject.register
|
29
35
|
subject.filter(event)
|
30
36
|
expect(event.get("translation")).to eq("OK")
|
31
37
|
end
|
32
38
|
end
|
33
39
|
|
40
|
+
describe "translation fails when regex setting is false but keys are regex based" do
|
41
|
+
|
42
|
+
let(:config) do
|
43
|
+
{
|
44
|
+
"field" => "status",
|
45
|
+
"destination" => "translation",
|
46
|
+
"dictionary" => [ "^2\\d\\d", "OK",
|
47
|
+
"^3\\d\\d", "Redirect",
|
48
|
+
"^4\\d\\d", "Client Error",
|
49
|
+
"^5\\d\\d", "Server Error" ],
|
50
|
+
"exact" => true,
|
51
|
+
"regex" => false
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
let(:event) { LogStash::Event.new("status" => 200) }
|
56
|
+
|
57
|
+
it "does not return the exact translation" do
|
58
|
+
subject.register
|
59
|
+
subject.filter(event)
|
60
|
+
expect(event.get("translation")).to be_nil
|
61
|
+
end
|
62
|
+
end
|
34
63
|
|
35
64
|
describe "multi translation" do
|
36
65
|
context "when using an inline dictionary" do
|
@@ -57,7 +86,7 @@ describe LogStash::Filters::Translate do
|
|
57
86
|
end
|
58
87
|
|
59
88
|
context "when using a file based dictionary" do
|
60
|
-
let(:dictionary_path) {
|
89
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
|
61
90
|
let(:config) do
|
62
91
|
{
|
63
92
|
"field" => "status",
|
@@ -104,7 +133,7 @@ describe LogStash::Filters::Translate do
|
|
104
133
|
end
|
105
134
|
|
106
135
|
context "when using a file based dictionary" do
|
107
|
-
let(:dictionary_path) {
|
136
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("regex_dict.csv") }
|
108
137
|
let(:config) do
|
109
138
|
{
|
110
139
|
"field" => "status",
|
@@ -167,7 +196,7 @@ describe LogStash::Filters::Translate do
|
|
167
196
|
|
168
197
|
describe "loading a dictionary" do
|
169
198
|
|
170
|
-
let(:dictionary_path) {
|
199
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("dict-wrong.yml") }
|
171
200
|
|
172
201
|
let(:config) do
|
173
202
|
{
|
@@ -186,7 +215,7 @@ describe LogStash::Filters::Translate do
|
|
186
215
|
end
|
187
216
|
|
188
217
|
context "when using a yml file" do
|
189
|
-
let(:dictionary_path) {
|
218
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("dict.yml") }
|
190
219
|
let(:event) { LogStash::Event.new("status" => "a") }
|
191
220
|
|
192
221
|
it "return the exact translation" do
|
@@ -197,7 +226,7 @@ describe LogStash::Filters::Translate do
|
|
197
226
|
end
|
198
227
|
|
199
228
|
context "when using a map tagged yml file" do
|
200
|
-
let(:dictionary_path) {
|
229
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
|
201
230
|
let(:event) { LogStash::Event.new("status" => "six") }
|
202
231
|
|
203
232
|
it "return the exact translation" do
|
@@ -208,7 +237,7 @@ describe LogStash::Filters::Translate do
|
|
208
237
|
end
|
209
238
|
|
210
239
|
context "when using a omap tagged yml file" do
|
211
|
-
let(:dictionary_path) {
|
240
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("tag-omap-dict.yml") }
|
212
241
|
let(:event) { LogStash::Event.new("status" => "nine") }
|
213
242
|
|
214
243
|
it "return the exact translation" do
|
@@ -219,7 +248,7 @@ describe LogStash::Filters::Translate do
|
|
219
248
|
end
|
220
249
|
|
221
250
|
context "when using a json file" do
|
222
|
-
let(:dictionary_path) {
|
251
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("dict.json") }
|
223
252
|
let(:event) { LogStash::Event.new("status" => "b") }
|
224
253
|
|
225
254
|
it "return the exact translation" do
|
@@ -230,7 +259,7 @@ describe LogStash::Filters::Translate do
|
|
230
259
|
end
|
231
260
|
|
232
261
|
context "when using a csv file" do
|
233
|
-
let(:dictionary_path) {
|
262
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("dict.csv") }
|
234
263
|
let(:event) { LogStash::Event.new("status" => "c") }
|
235
264
|
|
236
265
|
it "return the exact translation" do
|
@@ -241,7 +270,7 @@ describe LogStash::Filters::Translate do
|
|
241
270
|
end
|
242
271
|
|
243
272
|
context "when using an unknown file" do
|
244
|
-
let(:dictionary_path) {
|
273
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("dict.other") }
|
245
274
|
|
246
275
|
it "raises error" do
|
247
276
|
expect { subject.register }.to raise_error(RuntimeError, /Dictionary #{dictionary_path} has a non valid format/)
|
@@ -250,20 +279,21 @@ describe LogStash::Filters::Translate do
|
|
250
279
|
end
|
251
280
|
|
252
281
|
describe "iterate_on functionality" do
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
end
|
282
|
+
let(:config) do
|
283
|
+
{
|
284
|
+
"iterate_on" => "foo",
|
285
|
+
"field" => iterate_on_field,
|
286
|
+
"destination" => "baz",
|
287
|
+
"fallback" => "nooo",
|
288
|
+
"dictionary_path" => dictionary_path,
|
289
|
+
# "override" => true,
|
290
|
+
"refresh_interval" => 0
|
291
|
+
}
|
292
|
+
end
|
293
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
|
266
294
|
|
295
|
+
describe "when iterate_on is the same as field, AKA array of values" do
|
296
|
+
let(:iterate_on_field) { "foo" }
|
267
297
|
let(:event) { LogStash::Event.new("foo" => ["nine","eight", "seven"]) }
|
268
298
|
it "adds a translation to destination array for each value in field array" do
|
269
299
|
subject.register
|
@@ -272,20 +302,19 @@ describe LogStash::Filters::Translate do
|
|
272
302
|
end
|
273
303
|
end
|
274
304
|
|
275
|
-
describe "when iterate_on is
|
276
|
-
let(:
|
277
|
-
let(:
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
"dictionary_path" => dictionary_path,
|
284
|
-
# "override" => true,
|
285
|
-
"refresh_interval" => 0
|
286
|
-
}
|
305
|
+
describe "when iterate_on is the same as field, AKA array of values, coerces integer elements to strings" do
|
306
|
+
let(:iterate_on_field) { "foo" }
|
307
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
|
308
|
+
let(:event) { LogStash::Event.new("foo" => [200, 300, 400]) }
|
309
|
+
it "adds a translation to destination array for each value in field array" do
|
310
|
+
subject.register
|
311
|
+
subject.filter(event)
|
312
|
+
expect(event.get("baz")).to eq(["OK","Redirect","Client Error"])
|
287
313
|
end
|
314
|
+
end
|
288
315
|
|
316
|
+
describe "when iterate_on is not the same as field, AKA array of objects" do
|
317
|
+
let(:iterate_on_field) { "bar" }
|
289
318
|
let(:event) { LogStash::Event.new("foo" => [{"bar"=>"two"},{"bar"=>"one"}, {"bar"=>"six"}]) }
|
290
319
|
it "adds a translation to each map" do
|
291
320
|
subject.register
|
@@ -295,10 +324,23 @@ describe LogStash::Filters::Translate do
|
|
295
324
|
expect(event.get("[foo][2][baz]")).to eq("val-6-1|val-6-2")
|
296
325
|
end
|
297
326
|
end
|
327
|
+
|
328
|
+
describe "when iterate_on is not the same as field, AKA array of objects, coerces integer values to strings" do
|
329
|
+
let(:iterate_on_field) { "bar" }
|
330
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("regex_union_dict.csv") }
|
331
|
+
let(:event) { LogStash::Event.new("foo" => [{"bar"=>200},{"bar"=>300}, {"bar"=>400}]) }
|
332
|
+
it "adds a translation to each map" do
|
333
|
+
subject.register
|
334
|
+
subject.filter(event)
|
335
|
+
expect(event.get("[foo][0][baz]")).to eq("OK")
|
336
|
+
expect(event.get("[foo][1][baz]")).to eq("Redirect")
|
337
|
+
expect(event.get("[foo][2][baz]")).to eq("Client Error")
|
338
|
+
end
|
339
|
+
end
|
298
340
|
end
|
299
341
|
|
300
342
|
describe "field and destination are the same (needs override)" do
|
301
|
-
let(:dictionary_path) {
|
343
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("tag-map-dict.yml") }
|
302
344
|
let(:config) do
|
303
345
|
{
|
304
346
|
"field" => "foo",
|
@@ -319,7 +361,7 @@ describe LogStash::Filters::Translate do
|
|
319
361
|
end
|
320
362
|
|
321
363
|
describe "general configuration" do
|
322
|
-
let(:dictionary_path) {
|
364
|
+
let(:dictionary_path) { TranslateUtil.build_fixture_path("dict.yml") }
|
323
365
|
let(:config) do
|
324
366
|
{
|
325
367
|
"field" => "random field",
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-translate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.2.
|
4
|
+
version: 3.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|