logstash-filter-rest 0.5.2 → 0.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +12 -0
- data/README.md +1 -1
- data/lib/logstash/filters/rest.rb +31 -25
- data/logstash-filter-rest.gemspec +2 -2
- data/spec/filters/rest_spec.rb +58 -15
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d1b60b569aa8944cc187b1119df43ed5ed5b6a87
|
4
|
+
data.tar.gz: d192256c5722bc8dd4508a0025d5efcbf636e7b1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b2a68e2559564025924b531cf04cf72e80b82d62bc2402b4799cc55f878e1bc13255be9ceb6cb45714344c970a6990902fb31c9ad63e08249877bcc16b8ef406
|
7
|
+
data.tar.gz: dc9eab7e46af8e7ff74f4d3edebcba5a2617c2c2653f086677131d39fa4e106736d25cad13098708a98da15a30735bf107faad2825c93485c1b51973a640860b
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,15 @@
|
|
1
|
+
## 0.5.4
|
2
|
+
|
3
|
+
- update `gemspec` to work with logstash 5.5
|
4
|
+
|
5
|
+
## 0.5.3
|
6
|
+
- freeze all instance variables
|
7
|
+
- fix parallel processing by creating a `deep_clone` for each event
|
8
|
+
- use `LogStash::Util.deep_clone` for object cloning
|
9
|
+
- only dump body as json, if json is enabled in config (default)
|
10
|
+
- delete empty target testcase, as catched by upper logstash `LogStash::ConfigurationError`
|
11
|
+
- fix `sprintf` find and merge for more complex structures
|
12
|
+
|
1
13
|
## 0.5.2
|
2
14
|
- Fix behavior, where a referenced field (`%{...}`) has `ruby` chars
|
3
15
|
(i.e., consisting of `:`)
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Logstash REST Filter [](https://travis-ci.org/gandalfb/logstash-filter-rest)
|
2
2
|
|
3
3
|
This is a filter plugin for [Logstash](https://github.com/elasticsearch/logstash).
|
4
4
|
|
@@ -18,6 +18,10 @@ end
|
|
18
18
|
|
19
19
|
# Monkey Patch Array with deep freeze
|
20
20
|
class Array
|
21
|
+
def compact
|
22
|
+
delete_if { |v| v.respond_to?(:each) ? v.compact.empty? : v.nil? }
|
23
|
+
end
|
24
|
+
|
21
25
|
def deep_freeze
|
22
26
|
each { |j| j.deep_freeze if j.respond_to? :deep_freeze }
|
23
27
|
freeze
|
@@ -127,11 +131,12 @@ class LogStash::Filters::Rest < LogStash::Filters::Base
|
|
127
131
|
public
|
128
132
|
|
129
133
|
def register
|
130
|
-
@request = normalize_request(@request)
|
134
|
+
@request = normalize_request(@request).deep_freeze
|
131
135
|
@sprintf_fields = find_sprintf(
|
132
|
-
|
136
|
+
LogStash::Util.deep_clone(@request)
|
133
137
|
).deep_freeze
|
134
|
-
@
|
138
|
+
@sprintf_needed = !@sprintf_fields.empty?
|
139
|
+
@target = normalize_target(@target).freeze
|
135
140
|
end # def register
|
136
141
|
|
137
142
|
private
|
@@ -208,8 +213,10 @@ class LogStash::Filters::Rest < LogStash::Filters::Base
|
|
208
213
|
private
|
209
214
|
|
210
215
|
def request_http(request)
|
211
|
-
|
212
|
-
|
216
|
+
if request[2].key?(:body) && @json
|
217
|
+
request[2][:body] = LogStash::Json.dump(request[2][:body])
|
218
|
+
end
|
219
|
+
@logger.debug? && @logger.debug('fetching request',
|
213
220
|
:request => request)
|
214
221
|
|
215
222
|
method, url, *request_opts = request
|
@@ -247,9 +254,9 @@ class LogStash::Filters::Rest < LogStash::Filters::Base
|
|
247
254
|
private
|
248
255
|
|
249
256
|
def field_intrpl(intrpl_fields, event)
|
250
|
-
return intrpl_fields if intrpl_fields.empty?
|
251
|
-
return event.sprintf(intrpl_fields) unless intrpl_fields.respond_to?(:each)
|
252
257
|
case intrpl_fields
|
258
|
+
when String
|
259
|
+
result = event.sprintf(intrpl_fields)
|
253
260
|
when Array
|
254
261
|
result = []
|
255
262
|
intrpl_fields.each do |v|
|
@@ -260,6 +267,8 @@ class LogStash::Filters::Rest < LogStash::Filters::Base
|
|
260
267
|
intrpl_fields.each do |k, v|
|
261
268
|
result[k] = field_intrpl(v, event)
|
262
269
|
end
|
270
|
+
else
|
271
|
+
result = intrpl_fields
|
263
272
|
end
|
264
273
|
result
|
265
274
|
end
|
@@ -268,43 +277,40 @@ class LogStash::Filters::Rest < LogStash::Filters::Base
|
|
268
277
|
|
269
278
|
def filter(event)
|
270
279
|
return unless filter?(event)
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
end
|
280
|
+
request = LogStash::Util.deep_clone(@request)
|
281
|
+
@logger.debug? && @logger.debug('processing request',
|
282
|
+
:request => request,
|
283
|
+
:sprintf_needed => @sprintf_needed)
|
284
|
+
|
285
|
+
if @sprintf_needed
|
286
|
+
request = field_intrpl(request, event)
|
287
|
+
@logger.debug? && @logger.debug('interpolated request',
|
288
|
+
:request => request)
|
281
289
|
end
|
282
|
-
@logger.debug? && @logger.debug('Parsed request',
|
283
|
-
:request => @request)
|
284
290
|
|
285
291
|
client_error = nil
|
286
292
|
begin
|
287
|
-
code, body = request_http(
|
293
|
+
code, body = request_http(request)
|
288
294
|
rescue StandardError => e
|
289
295
|
client_error = e
|
290
296
|
end
|
291
297
|
|
292
298
|
if !client_error && code.between?(200, 299)
|
293
|
-
@logger.debug? && @logger.debug('
|
299
|
+
@logger.debug? && @logger.debug('success received',
|
294
300
|
:code => code, :body => body)
|
295
301
|
process_response(body, event)
|
296
302
|
else
|
297
|
-
@logger.debug? && @logger.debug('
|
303
|
+
@logger.debug? && @logger.debug('http error received',
|
298
304
|
:code => code, :body => body,
|
299
305
|
:client_error => client_error)
|
300
306
|
if @fallback.empty?
|
301
|
-
@logger.error('
|
302
|
-
:request =>
|
307
|
+
@logger.error('error in rest filter',
|
308
|
+
:request => request, :json => @json,
|
303
309
|
:code => code, :body => body,
|
304
310
|
:client_error => client_error)
|
305
311
|
@tag_on_rest_failure.each { |tag| event.tag(tag) }
|
306
312
|
else
|
307
|
-
@logger.debug? && @logger.debug('
|
313
|
+
@logger.debug? && @logger.debug('setting fallback',
|
308
314
|
:fallback => @fallback)
|
309
315
|
event.set(@target, @fallback)
|
310
316
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-rest'
|
3
|
-
s.version = '0.5.
|
3
|
+
s.version = '0.5.4'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = 'This filter requests data from a RESTful Web Service.'
|
6
6
|
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-rest. This gem is not a stand-alone program'
|
@@ -27,7 +27,7 @@ Gem::Specification.new do |s|
|
|
27
27
|
|
28
28
|
# Gem dependencies
|
29
29
|
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
|
30
|
-
s.add_runtime_dependency 'logstash-mixin-http_client', '>=
|
30
|
+
s.add_runtime_dependency 'logstash-mixin-http_client', '>= 5.0.0', '< 6.0.0'
|
31
31
|
|
32
32
|
s.add_development_dependency 'logstash-devutils', '>= 0', '< 2.0.0'
|
33
33
|
end
|
data/spec/filters/rest_spec.rb
CHANGED
@@ -277,17 +277,46 @@ describe LogStash::Filters::Rest do
|
|
277
277
|
expect(subject.get('rest')).to_not include('fallback')
|
278
278
|
end
|
279
279
|
end
|
280
|
-
describe '
|
280
|
+
describe 'Set to Rest Filter Post with body sprintf nested params' do
|
281
281
|
let(:config) do <<-CONFIG
|
282
282
|
filter {
|
283
283
|
rest {
|
284
284
|
request => {
|
285
|
-
url => '
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
285
|
+
url => 'https://jsonplaceholder.typicode.com/posts'
|
286
|
+
method => 'post'
|
287
|
+
body => {
|
288
|
+
key1 => [
|
289
|
+
{
|
290
|
+
"filterType" => "text"
|
291
|
+
"text" => "salmon"
|
292
|
+
"boolean" => false
|
293
|
+
},
|
294
|
+
{
|
295
|
+
"filterType" => "unique"
|
296
|
+
}
|
297
|
+
]
|
298
|
+
key2 => [
|
299
|
+
{
|
300
|
+
"message" => "123%{message}"
|
301
|
+
"boolean" => true
|
302
|
+
}
|
303
|
+
]
|
304
|
+
key3 => [
|
305
|
+
{
|
306
|
+
"text" => "%{message}123"
|
307
|
+
"filterType" => "text"
|
308
|
+
"number" => 44
|
309
|
+
},
|
310
|
+
{
|
311
|
+
"filterType" => "unique"
|
312
|
+
"null" => nil
|
313
|
+
}
|
314
|
+
]
|
315
|
+
userId => "%{message}"
|
316
|
+
}
|
317
|
+
headers => {
|
318
|
+
'Content-Type' => 'application/json'
|
319
|
+
}
|
291
320
|
}
|
292
321
|
target => 'rest'
|
293
322
|
}
|
@@ -295,14 +324,23 @@ describe LogStash::Filters::Rest do
|
|
295
324
|
CONFIG
|
296
325
|
end
|
297
326
|
|
298
|
-
sample('message' => '
|
327
|
+
sample('message' => '42') do
|
299
328
|
expect(subject).to include('rest')
|
300
|
-
expect(subject.get('rest')).to include('
|
301
|
-
expect(subject.get('rest')).to
|
302
|
-
expect(subject.get('rest')).
|
329
|
+
expect(subject.get('rest')).to include('key1')
|
330
|
+
expect(subject.get('[rest][key1][0][boolean]')).to eq('false')
|
331
|
+
expect(subject.get('[rest][key1][1][filterType]')).to eq('unique')
|
332
|
+
expect(subject.get('[rest][key2][0][message]')).to eq('12342')
|
333
|
+
expect(subject.get('[rest][key2][0][boolean]')).to eq('true')
|
334
|
+
expect(subject.get('[rest][key3][0][text]')).to eq('42123')
|
335
|
+
expect(subject.get('[rest][key3][0][filterType]')).to eq('text')
|
336
|
+
expect(subject.get('[rest][key3][0][number]')).to eq(44)
|
337
|
+
expect(subject.get('[rest][key3][1][filterType]')).to eq('unique')
|
338
|
+
expect(subject.get('[rest][key3][1][null]')).to eq('nil')
|
339
|
+
expect(subject.get('[rest][userId]')).to eq(42)
|
340
|
+
expect(subject.get('rest')).to_not include('fallback')
|
303
341
|
end
|
304
342
|
end
|
305
|
-
describe '
|
343
|
+
describe 'fallback' do
|
306
344
|
let(:config) do <<-CONFIG
|
307
345
|
filter {
|
308
346
|
rest {
|
@@ -314,16 +352,20 @@ describe LogStash::Filters::Rest do
|
|
314
352
|
'fallback1' => true
|
315
353
|
'fallback2' => true
|
316
354
|
}
|
317
|
-
target => ''
|
355
|
+
target => 'rest'
|
318
356
|
}
|
319
357
|
}
|
320
358
|
CONFIG
|
321
359
|
end
|
360
|
+
|
322
361
|
sample('message' => 'some text') do
|
323
|
-
expect
|
362
|
+
expect(subject).to include('rest')
|
363
|
+
expect(subject.get('rest')).to include('fallback1')
|
364
|
+
expect(subject.get('rest')).to include('fallback2')
|
365
|
+
expect(subject.get('rest')).to_not include('id')
|
324
366
|
end
|
325
367
|
end
|
326
|
-
describe '
|
368
|
+
describe 'empty target exception' do
|
327
369
|
let(:config) do <<-CONFIG
|
328
370
|
filter {
|
329
371
|
rest {
|
@@ -335,6 +377,7 @@ describe LogStash::Filters::Rest do
|
|
335
377
|
'fallback1' => true
|
336
378
|
'fallback2' => true
|
337
379
|
}
|
380
|
+
target => ''
|
338
381
|
}
|
339
382
|
}
|
340
383
|
CONFIG
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-rest
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lucas Henning
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-07-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -36,10 +36,10 @@ dependencies:
|
|
36
36
|
requirements:
|
37
37
|
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
39
|
+
version: 5.0.0
|
40
40
|
- - "<"
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version:
|
42
|
+
version: 6.0.0
|
43
43
|
name: logstash-mixin-http_client
|
44
44
|
prerelease: false
|
45
45
|
type: :runtime
|
@@ -47,10 +47,10 @@ dependencies:
|
|
47
47
|
requirements:
|
48
48
|
- - ">="
|
49
49
|
- !ruby/object:Gem::Version
|
50
|
-
version:
|
50
|
+
version: 5.0.0
|
51
51
|
- - "<"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
53
|
+
version: 6.0.0
|
54
54
|
- !ruby/object:Gem::Dependency
|
55
55
|
requirement: !ruby/object:Gem::Requirement
|
56
56
|
requirements:
|