logstash-filter-rest 0.1.6 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +27 -16
- data/lib/logstash/filters/rest.rb +211 -82
- data/logstash-filter-rest.gemspec +15 -13
- data/spec/filters/rest_spec.rb +249 -5
- data/spec/spec_helper.rb +2 -1
- metadata +54 -28
- data/Rakefile +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 70cf7db34fec1104a8eb742ce5633c11769a66d5
|
4
|
+
data.tar.gz: c265edbd76879148168331adc875befb458ec0a6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fac6c51358cd5cd6e9990807f03b169aeca591ea828c11f1b718b39639e99770edd534f896aa0b0ec1da8ed3e441d7807f34f8cfdf26e1403c75d73eeeaddbcd
|
7
|
+
data.tar.gz: 70a3d59b3d30ebc070e715f97be104c4e2161b59405b5ac488963a78d1eb8b400dd7b1d71025c8d382ca52d34d13de56401331793089a37d200c9d85872d5e86
|
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Logstash REST Filter
|
1
|
+
# Logstash REST Filter [![Build Status](https://travis-ci.org/gandalfb/logstash-filter-rest.svg?branch=http-keep-alive)](https://travis-ci.org/gandalfb/logstash-filter-rest)
|
2
2
|
|
3
3
|
This is a filter plugin for [Logstash](https://github.com/elasticsearch/logstash).
|
4
4
|
|
@@ -27,22 +27,33 @@ $LS_HOME/bin/plugin install logstash-filter-rest-0.1.0.gem
|
|
27
27
|
Add the following inside the filter section of your logstash configuration:
|
28
28
|
|
29
29
|
```sh
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
30
|
+
filter {
|
31
|
+
rest {
|
32
|
+
request => {
|
33
|
+
url => "http://example.com" # string (required, with field reference: "http://example.com?id=%{id}" or params, if defined)
|
34
|
+
method => "post" # string (optional, default = "get")
|
35
|
+
headers => { # hash (optional)
|
36
|
+
"key1" => "value1"
|
37
|
+
"key2" => "value2"
|
38
|
+
}
|
39
|
+
auth => {
|
40
|
+
user => "AzureDiamond"
|
41
|
+
password => "hunter2"
|
42
|
+
}
|
43
|
+
params => { # hash (optional, available for method => "get" and "post"; if post it will be transformed into body hash and posted as json)
|
44
|
+
"key1" => "value1"
|
45
|
+
"key2" => "value2"
|
46
|
+
"key3" => "%{somefield}" # Please set sprintf to true if you want to use field references
|
47
|
+
}
|
48
|
+
}
|
49
|
+
json => true # boolean (optional, default = false)
|
50
|
+
sprintf => true # boolean (optional, default = false, set this to true if you want to use field references in url, header or params)
|
51
|
+
target => "my_key" # string (optional, default = "rest_response")
|
52
|
+
fallback => { # hash describing a default in case of error
|
53
|
+
"key1" => "value1"
|
54
|
+
"key2" => "value2"
|
55
|
+
}
|
39
56
|
}
|
40
|
-
params => { # hash (optional, only available for method => "post")
|
41
|
-
"key1" => "value1"
|
42
|
-
"key2" => "value2"
|
43
|
-
"key3" => "%{somefield}" # Please set sprintf to true if you want to use field references
|
44
|
-
}
|
45
|
-
response_key => "my_key" # string (optional, default = "rest_response")
|
46
57
|
}
|
47
58
|
```
|
48
59
|
|
@@ -1,105 +1,234 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
2
|
+
require 'logstash/filters/base'
|
3
|
+
require 'logstash/namespace'
|
4
|
+
require 'logstash/plugin_mixins/http_client'
|
5
|
+
require 'logstash/json'
|
6
6
|
|
7
7
|
# Logstash REST Filter
|
8
8
|
# This filter calls a defined URL and saves the answer into a specified field.
|
9
9
|
#
|
10
10
|
class LogStash::Filters::Rest < LogStash::Filters::Base
|
11
|
+
include LogStash::PluginMixins::HttpClient
|
11
12
|
|
12
|
-
|
13
|
+
config_name 'rest'
|
14
|
+
|
15
|
+
# Configure the rest request send via HttpClient Plugin
|
16
|
+
# with hash objects used by the mixin plugin
|
17
|
+
#
|
18
|
+
# For example, if you want the data to be put in the `doc` field:
|
19
|
+
# [source,ruby]
|
20
|
+
# filter {
|
21
|
+
# rest {
|
22
|
+
# request => {
|
23
|
+
# url => "http://example.com" # string (required, with field reference: "http://example.com?id=%{id}" or params, if defined)
|
24
|
+
# method => "post" # string (optional, default = "get")
|
25
|
+
# headers => { # hash (optional)
|
26
|
+
# "key1" => "value1"
|
27
|
+
# "key2" => "value2"
|
28
|
+
# }
|
29
|
+
# auth => {
|
30
|
+
# user => "AzureDiamond"
|
31
|
+
# password => "hunter2"
|
32
|
+
# }
|
33
|
+
# params => { # hash (optional, available for method => "get" and "post"; if post it will be transformed into body hash and posted as json)
|
34
|
+
# "key1" => "value1"
|
35
|
+
# "key2" => "value2"
|
36
|
+
# "key3" => "%{somefield}" # Please set sprintf to true if you want to use field references
|
37
|
+
# }
|
38
|
+
# }
|
39
|
+
# }
|
40
|
+
# }
|
41
|
+
#
|
42
|
+
# NOTE: for further details, please reference https://github.com/logstash-plugins/logstash-mixin-http_client[logstash-mixin-http_client]
|
43
|
+
config :request, :validate => :hash, :required => true
|
44
|
+
|
45
|
+
# The plugin is written json centric, which defaults to true
|
46
|
+
# the response body will be parsed to json if true
|
13
47
|
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
18
|
-
#
|
19
|
-
#
|
20
|
-
#
|
21
|
-
|
22
|
-
|
23
|
-
#
|
24
|
-
#
|
25
|
-
# "key1" => "value1"
|
26
|
-
# "key2" => "value2"
|
27
|
-
# "key3" => "%{somefield}" # Please set sprintf to true if you want to use field references
|
28
|
-
# }
|
29
|
-
# response_key => "my_key" # string (optional, default = "rest_response")
|
30
|
-
# }
|
48
|
+
# [source,ruby]
|
49
|
+
# filter {
|
50
|
+
# rest {
|
51
|
+
# request => { .. }
|
52
|
+
# json => true
|
53
|
+
# }
|
54
|
+
# }
|
55
|
+
config :json, :validate => :boolean, :default => true
|
56
|
+
|
57
|
+
# If true, references to event fields can be made in
|
58
|
+
# url, params or body by using '%{somefield}'
|
31
59
|
#
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
60
|
+
# [source,ruby]
|
61
|
+
# filter {
|
62
|
+
# rest {
|
63
|
+
# request => { .. }
|
64
|
+
# sprintf => true
|
65
|
+
# }
|
66
|
+
# }
|
38
67
|
config :sprintf, :validate => :boolean, :default => false
|
39
|
-
|
40
|
-
|
41
|
-
|
68
|
+
|
69
|
+
# Defines the field, where the parsed response is written to
|
70
|
+
# if set to '' it will be written to event root
|
71
|
+
#
|
72
|
+
# For example, if you want the data to be put in the `doc` field:
|
73
|
+
# [source,ruby]
|
74
|
+
# filter {
|
75
|
+
# rest {
|
76
|
+
# request => { .. }
|
77
|
+
# target => "doc"
|
78
|
+
# }
|
79
|
+
# }
|
80
|
+
#
|
81
|
+
# NOTE: if the `target` field already exists, it will be overwritten!
|
82
|
+
config :target, :validate => :string, :default => 'rest'
|
83
|
+
|
84
|
+
# If set, any error like json parsing or invalid http response
|
85
|
+
# will result in this hash to be added to target instead of error tags
|
86
|
+
#
|
87
|
+
# For example, if you want the fallback data to be put in the `target` field:
|
88
|
+
# [source,ruby]
|
89
|
+
# filter {
|
90
|
+
# rest {
|
91
|
+
# request => { .. }
|
92
|
+
# fallback => {
|
93
|
+
# 'key1' => 'value1'
|
94
|
+
# 'key2' => 'value2'
|
95
|
+
# ...
|
96
|
+
# }
|
97
|
+
# }
|
98
|
+
# }
|
99
|
+
config :fallback, :validate => :hash, :default => {}
|
100
|
+
|
101
|
+
# Append values to the `tags` field when there has been no
|
102
|
+
# successful match or json parsing error
|
103
|
+
config :tag_on_rest_failure, :validate => :array, :default => ['_restfailure']
|
104
|
+
config :tag_on_json_failure, :validate => :array, :default => ['_jsonparsefailure']
|
42
105
|
|
43
106
|
public
|
44
|
-
def register
|
45
107
|
|
108
|
+
def register
|
109
|
+
@request = normalize_request(@request)
|
46
110
|
end # def register
|
47
111
|
|
112
|
+
private
|
113
|
+
|
114
|
+
def normalize_request(url_or_spec)
|
115
|
+
if url_or_spec.is_a?(String)
|
116
|
+
res = [:get, url_or_spec]
|
117
|
+
elsif url_or_spec.is_a?(Hash)
|
118
|
+
# The client will expect keys / values
|
119
|
+
spec = Hash[url_or_spec.clone.map { |k, v| [k.to_sym, v] }]
|
120
|
+
|
121
|
+
# method and url aren't really part of the options, so we pull them out
|
122
|
+
method = (spec.delete(:method) || :get).to_sym.downcase
|
123
|
+
url = spec.delete(:url)
|
124
|
+
|
125
|
+
# if it is a post and json, it is used as body string, not params
|
126
|
+
spec[:body] = spec.delete(:params) if method == :post
|
127
|
+
|
128
|
+
# We need these strings to be keywords!
|
129
|
+
spec[:auth] = { user: spec[:auth]['user'], pass: spec[:auth]['password'] } if spec[:auth]
|
130
|
+
|
131
|
+
res = [method, url, spec]
|
132
|
+
else
|
133
|
+
raise LogStash::ConfigurationError, "Invalid URL or request spec: '#{url_or_spec}', expected a String or Hash!"
|
134
|
+
end
|
135
|
+
|
136
|
+
validate_request!(url_or_spec, res)
|
137
|
+
res
|
138
|
+
end
|
139
|
+
|
140
|
+
private
|
141
|
+
|
142
|
+
def validate_request!(url_or_spec, request)
|
143
|
+
method, url, spec = request
|
144
|
+
|
145
|
+
raise LogStash::ConfigurationError, "No URL provided for request! #{url_or_spec}" unless url
|
146
|
+
raise LogStash::ConfigurationError, "Not supported request method #{method}" unless [ :get, :post ].include?( method )
|
147
|
+
|
148
|
+
if spec && spec[:auth]
|
149
|
+
raise LogStash::ConfigurationError, "Auth was specified, but 'user' was not!" unless spec[:auth][:user]
|
150
|
+
raise LogStash::ConfigurationError, "Auth was specified, but 'password' was not!" unless spec[:auth][:pass]
|
151
|
+
end
|
152
|
+
|
153
|
+
request
|
154
|
+
end
|
155
|
+
|
156
|
+
private
|
157
|
+
|
158
|
+
def request_http(request)
|
159
|
+
@logger.debug? && @logger.debug('Fetching URL', :request => request)
|
160
|
+
|
161
|
+
request[2][:body] = LogStash::Json.dump(request[2][:body]) if request[2].key?(:body)
|
162
|
+
|
163
|
+
method, url, *request_opts = request
|
164
|
+
response = client.http(method, url, *request_opts)
|
165
|
+
[response.code, response.body]
|
166
|
+
end
|
167
|
+
|
168
|
+
private
|
169
|
+
|
170
|
+
def process_response(response, event)
|
171
|
+
if @json
|
172
|
+
begin
|
173
|
+
parsed = LogStash::Json.load(response)
|
174
|
+
event = add_to_event(parsed, event)
|
175
|
+
rescue
|
176
|
+
if @fallback.empty?
|
177
|
+
@tag_on_json_failure.each { |tag| event.tag(tag) }
|
178
|
+
@logger.warn('JSON parsing error', :response => response, :event => event)
|
179
|
+
else
|
180
|
+
event = add_to_event(@fallback, event)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
else
|
184
|
+
event.set(@target, response.strip)
|
185
|
+
end
|
186
|
+
event
|
187
|
+
end
|
188
|
+
|
48
189
|
public
|
190
|
+
|
49
191
|
def filter(event)
|
50
192
|
return unless filter?(event)
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
event[response_key] = { }
|
71
|
-
event[response_key] = h
|
72
|
-
end
|
73
|
-
rescue
|
74
|
-
event['jsonerror'] = "unable to parse json"
|
75
|
-
end
|
76
|
-
else
|
77
|
-
event[@response_key] = response.strip
|
78
|
-
end
|
79
|
-
rescue
|
80
|
-
@logger.error("Error in Rest Filter. Parameters:", :url => url, :method => method, :json => json, :header => header, :params => params)
|
81
|
-
@logger.error("Rest Error Message:", :message => $!.message)
|
82
|
-
@logger.error("Backtrace:", :backtrace => $!.backtrace)
|
83
|
-
event['resterror'] = "Rest Filter Error. Please see Logstash Error Log for further information."
|
84
|
-
end
|
85
|
-
|
86
|
-
filter_matched(event)
|
193
|
+
@request[2][:params] = sprint(@sprintf, @request[2][:params], event) if @request[2].key?(:params)
|
194
|
+
@request[2][:body] = sprint(@sprintf, @request[2][:body], event) if @request[2].key?(:body)
|
195
|
+
@request[1] = sprint(@sprintf, @request[1], event)
|
196
|
+
|
197
|
+
code, body = request_http(@request)
|
198
|
+
if code.between?(200, 299)
|
199
|
+
event = process_response(body, event)
|
200
|
+
@logger.debug? && @logger.debug('Sucess received', :code => code, :body => body)
|
201
|
+
else
|
202
|
+
@logger.debug? && @logger.debug('Http error received', :code => code, :body => body)
|
203
|
+
if @fallback.empty?
|
204
|
+
@tag_on_rest_failure.each { |tag| event.tag(tag) }
|
205
|
+
@logger.error('Error in Rest filter', :request => @request, :json => @json, :code => code, :body => body)
|
206
|
+
else
|
207
|
+
event = add_to_event(@fallback, event)
|
208
|
+
@logger.debug? && @logger.debug('Setting fallback', :fallback => @fallback)
|
209
|
+
end
|
210
|
+
end
|
211
|
+
filter_matched(event)
|
87
212
|
end # def filter
|
88
|
-
|
213
|
+
|
214
|
+
private
|
215
|
+
|
89
216
|
def sprint(sprintf, hash, event)
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
end
|
96
|
-
return result
|
97
|
-
else
|
98
|
-
return event.sprintf(hash)
|
99
|
-
end
|
100
|
-
else
|
101
|
-
return hash
|
102
|
-
end
|
217
|
+
return hash unless sprintf
|
218
|
+
return event.sprintf(hash) unless hash.is_a?(Hash)
|
219
|
+
result = {}
|
220
|
+
hash.each { |k, v| result[k] = event.sprintf(v) }
|
221
|
+
result
|
103
222
|
end
|
104
223
|
|
224
|
+
private
|
225
|
+
|
226
|
+
def add_to_event(to_add, event)
|
227
|
+
if @target.empty?
|
228
|
+
to_add.each { |k, v| event[k] = v }
|
229
|
+
else
|
230
|
+
event[@target] = to_add
|
231
|
+
end
|
232
|
+
event
|
233
|
+
end
|
105
234
|
end # class LogStash::Filters::Rest
|
@@ -1,25 +1,27 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-rest'
|
3
|
-
s.version
|
3
|
+
s.version = '0.2.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
|
-
s.summary =
|
6
|
-
s.description =
|
7
|
-
s.authors = [
|
5
|
+
s.summary = 'This filter requests data from a RESTful Web Service.'
|
6
|
+
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
|
7
|
+
s.authors = ['Lucas Henning', 'Gandalf Buscher']
|
8
8
|
s.email = 'mail@hurb.de'
|
9
|
-
s.homepage =
|
10
|
-
s.require_paths = [
|
9
|
+
s.homepage = 'https://github.com/lucashenning/logstash-filter-rest/'
|
10
|
+
s.require_paths = ['lib']
|
11
11
|
|
12
12
|
# Files
|
13
|
-
s.files =
|
14
|
-
|
13
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
14
|
+
# Tests
|
15
15
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
16
|
|
17
17
|
# Special flag to let us know this is actually a logstash plugin
|
18
|
-
s.metadata = {
|
18
|
+
s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'filter' }
|
19
19
|
|
20
20
|
# Gem dependencies
|
21
|
-
s.add_runtime_dependency
|
22
|
-
s.add_runtime_dependency
|
23
|
-
s.add_runtime_dependency 'logstash-
|
24
|
-
|
21
|
+
s.add_runtime_dependency 'logstash-core', '>= 1.6.0', '< 3.0.0'
|
22
|
+
s.add_runtime_dependency 'logstash-codec-json', '>= 1.6.0', '< 3.0.0'
|
23
|
+
s.add_runtime_dependency 'logstash-mixin-http_client', '>= 2.2.4', '< 5.0.0'
|
24
|
+
|
25
|
+
s.add_development_dependency 'logstash-devutils', '~> 0'
|
26
|
+
s.add_development_dependency 'pry', '~> 0'
|
25
27
|
end
|
data/spec/filters/rest_spec.rb
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require
|
2
|
+
require 'logstash/filters/rest'
|
3
3
|
|
4
4
|
describe LogStash::Filters::Rest do
|
5
|
-
describe "Set to Rest Filter" do
|
5
|
+
describe "Set to Rest Filter Get without params" do
|
6
6
|
let(:config) do <<-CONFIG
|
7
7
|
filter {
|
8
8
|
rest {
|
9
|
-
|
9
|
+
request => {
|
10
|
+
url => "http://jsonplaceholder.typicode.com/users/10"
|
11
|
+
}
|
10
12
|
json => true
|
11
13
|
}
|
12
14
|
}
|
@@ -14,8 +16,250 @@ describe LogStash::Filters::Rest do
|
|
14
16
|
end
|
15
17
|
|
16
18
|
sample("message" => "some text") do
|
17
|
-
expect(subject).to include(
|
18
|
-
expect(subject['
|
19
|
+
expect(subject).to include('rest')
|
20
|
+
expect(subject['rest']).to include("id")
|
21
|
+
expect(subject['rest']['id']).to eq(10)
|
22
|
+
expect(subject['rest']).to_not include("fallback")
|
23
|
+
end
|
24
|
+
end
|
25
|
+
describe "Set to Rest Filter Get without params custom target" do
|
26
|
+
let(:config) do <<-CONFIG
|
27
|
+
filter {
|
28
|
+
rest {
|
29
|
+
request => {
|
30
|
+
url => "http://jsonplaceholder.typicode.com/users/10"
|
31
|
+
}
|
32
|
+
json => true
|
33
|
+
target => 'testing'
|
34
|
+
}
|
35
|
+
}
|
36
|
+
CONFIG
|
37
|
+
end
|
38
|
+
|
39
|
+
sample("message" => "some text") do
|
40
|
+
expect(subject).to include('testing')
|
41
|
+
expect(subject['testing']).to include("id")
|
42
|
+
expect(subject['testing']['id']).to eq(10)
|
43
|
+
expect(subject['testing']).to_not include("fallback")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
describe "Set to Rest Filter Get without params and sprintf" do
|
47
|
+
let(:config) do <<-CONFIG
|
48
|
+
filter {
|
49
|
+
rest {
|
50
|
+
request => {
|
51
|
+
url => "http://jsonplaceholder.typicode.com/users/%{message}"
|
52
|
+
}
|
53
|
+
json => true
|
54
|
+
sprintf => true
|
55
|
+
}
|
56
|
+
}
|
57
|
+
CONFIG
|
58
|
+
end
|
59
|
+
|
60
|
+
sample("message" => "10") do
|
61
|
+
expect(subject).to include('rest')
|
62
|
+
expect(subject['rest']).to include("id")
|
63
|
+
expect(subject['rest']['id']).to eq(10)
|
64
|
+
expect(subject['rest']).to_not include("fallback")
|
65
|
+
end
|
66
|
+
end
|
67
|
+
describe "Set to Rest Filter Get without params http error" do
|
68
|
+
let(:config) do <<-CONFIG
|
69
|
+
filter {
|
70
|
+
rest {
|
71
|
+
request => {
|
72
|
+
url => "http://httpstat.us/404"
|
73
|
+
}
|
74
|
+
json => true
|
75
|
+
}
|
76
|
+
}
|
77
|
+
CONFIG
|
78
|
+
end
|
79
|
+
|
80
|
+
sample("message" => "some text") do
|
81
|
+
expect(subject).to_not include('rest')
|
82
|
+
expect(subject['tags']).to include('_restfailure')
|
83
|
+
end
|
84
|
+
end
|
85
|
+
describe "Set to Rest Filter Get with params" do
|
86
|
+
let(:config) do <<-CONFIG
|
87
|
+
filter {
|
88
|
+
rest {
|
89
|
+
request => {
|
90
|
+
url => "https://jsonplaceholder.typicode.com/posts"
|
91
|
+
params => {
|
92
|
+
userId => 10
|
93
|
+
}
|
94
|
+
headers => {
|
95
|
+
"Content-Type" => "application/json"
|
96
|
+
}
|
97
|
+
}
|
98
|
+
json => true
|
99
|
+
}
|
100
|
+
}
|
101
|
+
CONFIG
|
102
|
+
end
|
103
|
+
|
104
|
+
sample("message" => "some text") do
|
105
|
+
expect(subject).to include('rest')
|
106
|
+
expect(subject['rest'][0]).to include("userId")
|
107
|
+
expect(subject['rest'][0]['userId']).to eq(10)
|
108
|
+
expect(subject['rest']).to_not include("fallback")
|
109
|
+
end
|
110
|
+
end
|
111
|
+
describe "Set to Rest Filter Get with params sprintf" do
|
112
|
+
let(:config) do <<-CONFIG
|
113
|
+
filter {
|
114
|
+
rest {
|
115
|
+
request => {
|
116
|
+
url => "https://jsonplaceholder.typicode.com/posts"
|
117
|
+
params => {
|
118
|
+
userId => "%{message}"
|
119
|
+
}
|
120
|
+
headers => {
|
121
|
+
"Content-Type" => "application/json"
|
122
|
+
}
|
123
|
+
}
|
124
|
+
json => true
|
125
|
+
sprintf => true
|
126
|
+
}
|
127
|
+
}
|
128
|
+
CONFIG
|
129
|
+
end
|
130
|
+
|
131
|
+
sample("message" => "10") do
|
132
|
+
expect(subject).to include('rest')
|
133
|
+
expect(subject['rest'][0]).to include("userId")
|
134
|
+
expect(subject['rest'][0]['userId']).to eq(10)
|
135
|
+
expect(subject['rest']).to_not include("fallback")
|
136
|
+
end
|
137
|
+
end
|
138
|
+
describe "Set to Rest Filter Post with params" do
|
139
|
+
let(:config) do <<-CONFIG
|
140
|
+
filter {
|
141
|
+
rest {
|
142
|
+
request => {
|
143
|
+
url => "https://jsonplaceholder.typicode.com/posts"
|
144
|
+
method => "post"
|
145
|
+
params => {
|
146
|
+
title => 'foo'
|
147
|
+
body => 'bar'
|
148
|
+
userId => 42
|
149
|
+
}
|
150
|
+
headers => {
|
151
|
+
"Content-Type" => "application/json"
|
152
|
+
}
|
153
|
+
}
|
154
|
+
json => true
|
155
|
+
}
|
156
|
+
}
|
157
|
+
CONFIG
|
158
|
+
end
|
159
|
+
|
160
|
+
sample("message" => "some text") do
|
161
|
+
expect(subject).to include('rest')
|
162
|
+
expect(subject['rest']).to include("id")
|
163
|
+
expect(subject['rest']['userId']).to eq(42)
|
164
|
+
expect(subject['rest']).to_not include("fallback")
|
165
|
+
end
|
166
|
+
end
|
167
|
+
describe "Set to Rest Filter Post with params sprintf" do
|
168
|
+
let(:config) do <<-CONFIG
|
169
|
+
filter {
|
170
|
+
rest {
|
171
|
+
request => {
|
172
|
+
url => "https://jsonplaceholder.typicode.com/posts"
|
173
|
+
method => "post"
|
174
|
+
params => {
|
175
|
+
title => 'foo'
|
176
|
+
body => 'bar'
|
177
|
+
userId => "%{message}"
|
178
|
+
}
|
179
|
+
headers => {
|
180
|
+
"Content-Type" => "application/json"
|
181
|
+
}
|
182
|
+
}
|
183
|
+
json => true
|
184
|
+
sprintf => true
|
185
|
+
}
|
186
|
+
}
|
187
|
+
CONFIG
|
188
|
+
end
|
189
|
+
|
190
|
+
sample("message" => "42") do
|
191
|
+
expect(subject).to include('rest')
|
192
|
+
expect(subject['rest']).to include("id")
|
193
|
+
expect(subject['rest']['userId']).to eq(42)
|
194
|
+
expect(subject['rest']).to_not include("fallback")
|
195
|
+
end
|
196
|
+
end
|
197
|
+
describe "Fallback" do
|
198
|
+
let(:config) do <<-CONFIG
|
199
|
+
filter {
|
200
|
+
rest {
|
201
|
+
request => {
|
202
|
+
url => "http://jsonplaceholder.typicode.com/users/0"
|
203
|
+
}
|
204
|
+
json => true
|
205
|
+
fallback => {
|
206
|
+
"fallback1" => true
|
207
|
+
"fallback2" => true
|
208
|
+
}
|
209
|
+
}
|
210
|
+
}
|
211
|
+
CONFIG
|
212
|
+
end
|
213
|
+
|
214
|
+
sample("message" => "some text") do
|
215
|
+
expect(subject).to include('rest')
|
216
|
+
expect(subject['rest']).to include("fallback1")
|
217
|
+
expect(subject['rest']).to include("fallback2")
|
218
|
+
expect(subject['rest']).to_not include("id")
|
219
|
+
end
|
220
|
+
end
|
221
|
+
describe "Fallback empty target" do
|
222
|
+
let(:config) do <<-CONFIG
|
223
|
+
filter {
|
224
|
+
rest {
|
225
|
+
request => {
|
226
|
+
url => "http://jsonplaceholder.typicode.com/users/0"
|
227
|
+
}
|
228
|
+
json => true
|
229
|
+
target => ''
|
230
|
+
fallback => {
|
231
|
+
"fallback1" => true
|
232
|
+
"fallback2" => true
|
233
|
+
}
|
234
|
+
}
|
235
|
+
}
|
236
|
+
CONFIG
|
237
|
+
end
|
238
|
+
|
239
|
+
sample("message" => "some text") do
|
240
|
+
expect(subject).to_not include('rest')
|
241
|
+
expect(subject).to include("fallback1")
|
242
|
+
expect(subject).to include("fallback2")
|
243
|
+
expect(subject).to_not include("id")
|
244
|
+
end
|
245
|
+
end
|
246
|
+
describe "Empty target" do
|
247
|
+
let(:config) do <<-CONFIG
|
248
|
+
filter {
|
249
|
+
rest {
|
250
|
+
request => {
|
251
|
+
url => "http://jsonplaceholder.typicode.com/users/1"
|
252
|
+
}
|
253
|
+
json => true
|
254
|
+
target => ''
|
255
|
+
}
|
256
|
+
}
|
257
|
+
CONFIG
|
258
|
+
end
|
259
|
+
|
260
|
+
sample("message" => "some text") do
|
261
|
+
expect(subject).to include("id")
|
262
|
+
expect(subject).to_not include("fallback")
|
19
263
|
end
|
20
264
|
end
|
21
265
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1 +1,2 @@
|
|
1
|
-
require
|
1
|
+
require 'logstash/devutils/rspec/spec_helper'
|
2
|
+
require 'pry'
|
metadata
CHANGED
@@ -1,77 +1,104 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-rest
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lucas Henning
|
8
|
+
- Gandalf Buscher
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2016-
|
12
|
+
date: 2016-08-20 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.6.0
|
20
|
+
- - "<"
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 3.0.0
|
14
23
|
name: logstash-core
|
24
|
+
prerelease: false
|
25
|
+
type: :runtime
|
15
26
|
version_requirements: !ruby/object:Gem::Requirement
|
16
27
|
requirements:
|
17
|
-
- -
|
28
|
+
- - ">="
|
18
29
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.
|
20
|
-
- - <
|
30
|
+
version: 1.6.0
|
31
|
+
- - "<"
|
21
32
|
- !ruby/object:Gem::Version
|
22
33
|
version: 3.0.0
|
34
|
+
- !ruby/object:Gem::Dependency
|
23
35
|
requirement: !ruby/object:Gem::Requirement
|
24
36
|
requirements:
|
25
|
-
- -
|
37
|
+
- - ">="
|
26
38
|
- !ruby/object:Gem::Version
|
27
|
-
version: 1.
|
28
|
-
- - <
|
39
|
+
version: 1.6.0
|
40
|
+
- - "<"
|
29
41
|
- !ruby/object:Gem::Version
|
30
42
|
version: 3.0.0
|
43
|
+
name: logstash-codec-json
|
31
44
|
prerelease: false
|
32
45
|
type: :runtime
|
33
|
-
- !ruby/object:Gem::Dependency
|
34
|
-
name: rest-client
|
35
46
|
version_requirements: !ruby/object:Gem::Requirement
|
36
47
|
requirements:
|
37
|
-
- -
|
48
|
+
- - ">="
|
38
49
|
- !ruby/object:Gem::Version
|
39
|
-
version: 1.
|
50
|
+
version: 1.6.0
|
51
|
+
- - "<"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 3.0.0
|
54
|
+
- !ruby/object:Gem::Dependency
|
40
55
|
requirement: !ruby/object:Gem::Requirement
|
41
56
|
requirements:
|
42
|
-
- -
|
57
|
+
- - ">="
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: 2.2.4
|
60
|
+
- - "<"
|
43
61
|
- !ruby/object:Gem::Version
|
44
|
-
version:
|
62
|
+
version: 5.0.0
|
63
|
+
name: logstash-mixin-http_client
|
45
64
|
prerelease: false
|
46
65
|
type: :runtime
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
|
-
name: logstash-codec-json
|
49
66
|
version_requirements: !ruby/object:Gem::Requirement
|
50
67
|
requirements:
|
51
|
-
- -
|
68
|
+
- - ">="
|
52
69
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
70
|
+
version: 2.2.4
|
71
|
+
- - "<"
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: 5.0.0
|
74
|
+
- !ruby/object:Gem::Dependency
|
54
75
|
requirement: !ruby/object:Gem::Requirement
|
55
76
|
requirements:
|
56
|
-
- -
|
77
|
+
- - "~>"
|
57
78
|
- !ruby/object:Gem::Version
|
58
79
|
version: '0'
|
59
|
-
prerelease: false
|
60
|
-
type: :runtime
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
80
|
name: logstash-devutils
|
81
|
+
prerelease: false
|
82
|
+
type: :development
|
63
83
|
version_requirements: !ruby/object:Gem::Requirement
|
64
84
|
requirements:
|
65
|
-
- -
|
85
|
+
- - "~>"
|
66
86
|
- !ruby/object:Gem::Version
|
67
87
|
version: '0'
|
88
|
+
- !ruby/object:Gem::Dependency
|
68
89
|
requirement: !ruby/object:Gem::Requirement
|
69
90
|
requirements:
|
70
|
-
- -
|
91
|
+
- - "~>"
|
71
92
|
- !ruby/object:Gem::Version
|
72
93
|
version: '0'
|
94
|
+
name: pry
|
73
95
|
prerelease: false
|
74
96
|
type: :development
|
97
|
+
version_requirements: !ruby/object:Gem::Requirement
|
98
|
+
requirements:
|
99
|
+
- - "~>"
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
75
102
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
76
103
|
email: mail@hurb.de
|
77
104
|
executables: []
|
@@ -82,7 +109,6 @@ files:
|
|
82
109
|
- Gemfile
|
83
110
|
- LICENSE
|
84
111
|
- README.md
|
85
|
-
- Rakefile
|
86
112
|
- lib/logstash/filters/rest.rb
|
87
113
|
- logstash-filter-rest.gemspec
|
88
114
|
- spec/filters/rest_spec.rb
|
@@ -99,17 +125,17 @@ require_paths:
|
|
99
125
|
- lib
|
100
126
|
required_ruby_version: !ruby/object:Gem::Requirement
|
101
127
|
requirements:
|
102
|
-
- -
|
128
|
+
- - ">="
|
103
129
|
- !ruby/object:Gem::Version
|
104
130
|
version: '0'
|
105
131
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
106
132
|
requirements:
|
107
|
-
- -
|
133
|
+
- - ">="
|
108
134
|
- !ruby/object:Gem::Version
|
109
135
|
version: '0'
|
110
136
|
requirements: []
|
111
137
|
rubyforge_project:
|
112
|
-
rubygems_version: 2.4
|
138
|
+
rubygems_version: 2.6.4
|
113
139
|
signing_key:
|
114
140
|
specification_version: 4
|
115
141
|
summary: This filter requests data from a RESTful Web Service.
|
data/Rakefile
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
require "logstash/devutils/rake"
|