logstash-filter-translate 3.0.4 → 3.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/LICENSE +1 -1
- data/docs/index.asciidoc +18 -5
- data/lib/logstash/filters/translate.rb +23 -8
- data/logstash-filter-translate.gemspec +1 -1
- data/spec/filters/translate_spec.rb +69 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5e4b8c724a742d47aa89a6237e127be569f2675a19c5752e719918bff8489c77
|
4
|
+
data.tar.gz: 406d7e0417251e10d1142d19027af9557f9a0993782bb16780bcdc86c943bd5e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e25757bc66a2b1afa15318161c6a8182f5258e9a8f395c870e0826224f6111bfa607950e678058a423aecb733f9f23edbbaed7778067f239269786afc76341d9
|
7
|
+
data.tar.gz: c84f492fad3418db7e18333658691fe5a8109c6dfc15589e88415198d348cce4fa05d4b966762072387441dec06210d6c7bf5fcc92fe5bec94ffe1b5026eba9d
|
data/CHANGELOG.md
CHANGED
data/LICENSE
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -60,6 +60,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
60
60
|
| <<plugins-{type}s-{plugin}-override>> |<<boolean,boolean>>|No
|
61
61
|
| <<plugins-{type}s-{plugin}-refresh_interval>> |<<number,number>>|No
|
62
62
|
| <<plugins-{type}s-{plugin}-regex>> |<<boolean,boolean>>|No
|
63
|
+
| <<plugins-{type}s-{plugin}-refresh_behaviour>> |<<string,string>>|No
|
63
64
|
|=======================================================================
|
64
65
|
|
65
66
|
Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
|
@@ -91,10 +92,12 @@ Example:
|
|
91
92
|
[source,ruby]
|
92
93
|
filter {
|
93
94
|
translate {
|
94
|
-
dictionary =>
|
95
|
-
|
96
|
-
|
97
|
-
|
95
|
+
dictionary => {
|
96
|
+
"100" => "Continue",
|
97
|
+
"101" => "Switching Protocols",
|
98
|
+
"merci" => "thank you",
|
99
|
+
"old version" => "new version"
|
100
|
+
}
|
98
101
|
}
|
99
102
|
}
|
100
103
|
|
@@ -205,7 +208,17 @@ When using a dictionary file, this setting will indicate how frequently
|
|
205
208
|
If you'd like to treat dictionary keys as regular expressions, set `exact => true`.
|
206
209
|
Note: this is activated only when `exact => true`.
|
207
210
|
|
211
|
+
[id="plugins-{type}s-{plugin}-refresh_behaviour"]
|
212
|
+
===== `refresh_behaviour`
|
213
|
+
|
214
|
+
* Value type is <<string,string>>
|
215
|
+
* Default value is `merge`
|
216
|
+
|
217
|
+
When using a dictionary file, this setting indicates how the update will be executed.
|
218
|
+
Setting this to `merge` leads to entries removed from the dictionary file being kept;
|
219
|
+
`replace` deletes old entries on update.
|
220
|
+
|
208
221
|
|
209
222
|
|
210
223
|
[id="plugins-{type}s-{plugin}-common-options"]
|
211
|
-
include::{include_path}/{type}.asciidoc[]
|
224
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -61,9 +61,11 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
|
|
61
61
|
# NOTE: It is an error to specify both `dictionary` and `dictionary_path`.
|
62
62
|
config :dictionary, :validate => :hash, :default => {}
|
63
63
|
|
64
|
-
# The full path of the external dictionary file. The format of the table
|
65
|
-
#
|
66
|
-
|
64
|
+
# The full path of the external dictionary file. The format of the table should
|
65
|
+
# be a standard YAML, JSON or CSV with filenames ending in `.yaml`, `.yml`,
|
66
|
+
#`.json` or `.csv` to be read. Make sure you specify any integer-based keys in
|
67
|
+
# quotes. For example, the YAML file (`.yaml` or `.yml` should look something like
|
68
|
+
# this:
|
67
69
|
# [source,ruby]
|
68
70
|
# "100": Continue
|
69
71
|
# "101": Switching Protocols
|
@@ -126,6 +128,11 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
|
|
126
128
|
# This configuration can be dynamic and include parts of the event using the `%{field}` syntax.
|
127
129
|
config :fallback, :validate => :string
|
128
130
|
|
131
|
+
# When using a dictionary file, this setting indicates how the update will be executed.
|
132
|
+
# Setting this to `merge` leads to entries removed from the dictionary file being kept; `replace`
|
133
|
+
# deletes old entries on update.
|
134
|
+
config :refresh_behaviour, :validate => ['merge', 'replace'], :default => 'merge'
|
135
|
+
|
129
136
|
def register
|
130
137
|
rw_lock = java.util.concurrent.locks.ReentrantReadWriteLock.new
|
131
138
|
@read_lock = rw_lock.readLock
|
@@ -243,7 +250,7 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
|
|
243
250
|
@logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
|
244
251
|
return
|
245
252
|
end
|
246
|
-
|
253
|
+
refresh_dictionary!(YAML.load_file(@dictionary_path))
|
247
254
|
end
|
248
255
|
|
249
256
|
def load_json(raise_exception=false)
|
@@ -251,7 +258,7 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
|
|
251
258
|
@logger.warn("dictionary file read failure, continuing with old dictionary", :path => @dictionary_path)
|
252
259
|
return
|
253
260
|
end
|
254
|
-
|
261
|
+
refresh_dictionary!(JSON.parse(File.read(@dictionary_path)))
|
255
262
|
end
|
256
263
|
|
257
264
|
def load_csv(raise_exception=false)
|
@@ -263,11 +270,19 @@ class LogStash::Filters::Translate < LogStash::Filters::Base
|
|
263
270
|
acc[v[0]] = v[1]
|
264
271
|
acc
|
265
272
|
end
|
266
|
-
|
273
|
+
refresh_dictionary!(data)
|
267
274
|
end
|
268
275
|
|
269
|
-
def
|
270
|
-
@
|
276
|
+
def refresh_dictionary!(data)
|
277
|
+
case @refresh_behaviour
|
278
|
+
when 'merge'
|
279
|
+
@dictionary.merge!(data)
|
280
|
+
when 'replace'
|
281
|
+
@dictionary = data
|
282
|
+
else
|
283
|
+
# we really should never get here
|
284
|
+
raise(LogStash::ConfigurationError, "Unknown value for refresh_behaviour=#{@refresh_behaviour.to_s}")
|
285
|
+
end
|
271
286
|
end
|
272
287
|
|
273
288
|
def loading_exception(e, raise_exception=false)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-translate'
|
4
|
-
s.version = '3.0
|
4
|
+
s.version = '3.1.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Replaces field contents based on a hash or YAML file"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -195,4 +195,73 @@ describe LogStash::Filters::Translate do
|
|
195
195
|
expect { subject.register }.to raise_error(LogStash::ConfigurationError)
|
196
196
|
end
|
197
197
|
end
|
198
|
+
|
199
|
+
describe "refresh_behaviour" do
|
200
|
+
let(:dictionary_content) { "a : 1\nb : 2\nc : 3" }
|
201
|
+
let(:modified_content) { "a : 1\nb : 4" }
|
202
|
+
let(:dictionary_path) { "#{Stud::Temporary.pathname}.yml" }
|
203
|
+
let(:refresh_behaviour) { "merge" }
|
204
|
+
let(:config) do
|
205
|
+
{
|
206
|
+
"field" => "status",
|
207
|
+
"destination" => "translation",
|
208
|
+
"dictionary_path" => dictionary_path,
|
209
|
+
"refresh_interval" => 10000, # we're controlling this manually
|
210
|
+
"exact" => true,
|
211
|
+
"regex" => false,
|
212
|
+
"fallback" => "no match",
|
213
|
+
"refresh_behaviour" => refresh_behaviour
|
214
|
+
}
|
215
|
+
end
|
216
|
+
|
217
|
+
before :each do
|
218
|
+
IO.write(dictionary_path, dictionary_content)
|
219
|
+
subject.register
|
220
|
+
end
|
221
|
+
|
222
|
+
let(:before_mod) { LogStash::Event.new("status" => "b") }
|
223
|
+
let(:after_mod) { LogStash::Event.new("status" => "b") }
|
224
|
+
let(:before_del) { LogStash::Event.new("status" => "c") }
|
225
|
+
let(:after_del) { LogStash::Event.new("status" => "c") }
|
226
|
+
|
227
|
+
context "when 'merge'" do
|
228
|
+
let(:refresh_behaviour) { 'merge' }
|
229
|
+
it "overwrites existing entries" do
|
230
|
+
subject.filter(before_mod)
|
231
|
+
IO.write(dictionary_path, modified_content)
|
232
|
+
subject.send(:load_dictionary)
|
233
|
+
subject.filter(after_mod)
|
234
|
+
expect(before_mod.get("translation")).to eq(2)
|
235
|
+
expect(after_mod.get("translation")).to eq(4)
|
236
|
+
end
|
237
|
+
it "keeps leftover entries" do
|
238
|
+
subject.filter(before_del)
|
239
|
+
IO.write(dictionary_path, modified_content)
|
240
|
+
subject.send(:load_dictionary)
|
241
|
+
subject.filter(after_del)
|
242
|
+
expect(before_del.get("translation")).to eq(3)
|
243
|
+
expect(after_del.get("translation")).to eq(3)
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
context "when 'replace'" do
|
248
|
+
let(:refresh_behaviour) { 'replace' }
|
249
|
+
it "overwrites existing entries" do
|
250
|
+
subject.filter(before_mod)
|
251
|
+
IO.write(dictionary_path, modified_content)
|
252
|
+
subject.send(:load_dictionary)
|
253
|
+
subject.filter(after_mod)
|
254
|
+
expect(before_mod.get("translation")).to eq(2)
|
255
|
+
expect(after_mod.get("translation")).to eq(4)
|
256
|
+
end
|
257
|
+
it "removes leftover entries" do
|
258
|
+
subject.filter(before_del)
|
259
|
+
IO.write(dictionary_path, modified_content)
|
260
|
+
subject.send(:load_dictionary)
|
261
|
+
subject.filter(after_del)
|
262
|
+
expect(before_del.get("translation")).to eq(3)
|
263
|
+
expect(after_del.get("translation")).to eq("no match")
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
198
267
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-translate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0
|
4
|
+
version: 3.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-04-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -89,7 +89,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
89
89
|
version: '0'
|
90
90
|
requirements: []
|
91
91
|
rubyforge_project:
|
92
|
-
rubygems_version: 2.6.
|
92
|
+
rubygems_version: 2.6.13
|
93
93
|
signing_key:
|
94
94
|
specification_version: 4
|
95
95
|
summary: Replaces field contents based on a hash or YAML file
|