logstash-filter-http 1.1.0 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/filters/http.rb +39 -3
- data/logstash-filter-http.gemspec +3 -1
- data/spec/filters/http_spec.rb +118 -52
- metadata +30 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 676e5abcdac3f6a16a8274eb18793cd7d75fe26545f7b2bfee9f707b6cf1fb90
|
4
|
+
data.tar.gz: 3ed6f080ba7323a07621cc69380f693a07b51ab5b374d0172d660a8682396f90
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 76adf00a0498d9f2f1d43b898457d744dce0a8ecef5bc15b559682513e2fc3cf09747925dea0a01b515762cbba9f9e5106ee316ea7c44a420a13b67596c24656
|
7
|
+
data.tar.gz: ceeaadb9afcc5678f87f7b1d3ccf6446d56826db73354655bc5f4976d0bf07698bc061db234fd27cf569de9fd1b467dbb6a75c918a04929918235ef42a08869e
|
data/CHANGELOG.md
CHANGED
@@ -1,13 +1,21 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require 'logstash/filters/base'
|
3
|
+
require 'logstash/json'
|
3
4
|
require 'logstash/namespace'
|
4
5
|
require 'logstash/plugin_mixins/http_client'
|
5
|
-
require 'logstash/
|
6
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support'
|
7
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
|
8
|
+
require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
|
6
9
|
|
7
10
|
# Logstash HTTP Filter
|
8
11
|
# This filter calls a defined URL and saves the answer into a specified field.
|
9
12
|
#
|
10
13
|
class LogStash::Filters::Http < LogStash::Filters::Base
|
14
|
+
|
15
|
+
include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
|
16
|
+
|
17
|
+
extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
|
18
|
+
|
11
19
|
include LogStash::PluginMixins::HttpClient
|
12
20
|
|
13
21
|
config_name 'http'
|
@@ -21,14 +29,28 @@ class LogStash::Filters::Http < LogStash::Filters::Base
|
|
21
29
|
config :body, :required => false
|
22
30
|
config :body_format, :validate => ['text', 'json'], :default => "text"
|
23
31
|
|
24
|
-
|
25
|
-
config :
|
32
|
+
# default [body] (legacy) required to be specified in ECS mode
|
33
|
+
config :target_body, :validate => :field_reference
|
34
|
+
# default [headers] (legacy) or [@metadata][filter][http][response][headers] in ECS mode
|
35
|
+
config :target_headers, :validate => :field_reference
|
26
36
|
|
27
37
|
# Append values to the `tags` field when there has been no
|
28
38
|
# successful match or json parsing error
|
29
39
|
config :tag_on_request_failure, :validate => :array, :default => ['_httprequestfailure']
|
30
40
|
config :tag_on_json_failure, :validate => :array, :default => ['_jsonparsefailure']
|
31
41
|
|
42
|
+
def initialize(*params)
|
43
|
+
super
|
44
|
+
|
45
|
+
@target_body ||= ecs_select[disabled: '[body]', v1: false]
|
46
|
+
if @target_body.eql? false # user needs to specify target in ECS mode
|
47
|
+
@logger.error missing_config_message(:target_body)
|
48
|
+
raise LogStash::ConfigurationError.new "Wrong configuration (in ecs_compatibility mode #{ecs_compatibility.inspect})"
|
49
|
+
end
|
50
|
+
|
51
|
+
@target_headers ||= ecs_select[disabled: '[headers]', v1: '[@metadata][filter][http][response][headers]']
|
52
|
+
end
|
53
|
+
|
32
54
|
def register
|
33
55
|
# nothing to see here
|
34
56
|
@verb = verb.downcase
|
@@ -79,6 +101,20 @@ class LogStash::Filters::Http < LogStash::Filters::Base
|
|
79
101
|
end # def filter
|
80
102
|
|
81
103
|
private
|
104
|
+
|
105
|
+
def missing_config_message(name)
|
106
|
+
<<-EOF
|
107
|
+
Missing a required setting for the http filter plugin:
|
108
|
+
|
109
|
+
filter {
|
110
|
+
http {
|
111
|
+
#{name} => # SETTING MISSING
|
112
|
+
...
|
113
|
+
}
|
114
|
+
}
|
115
|
+
EOF
|
116
|
+
end
|
117
|
+
|
82
118
|
def request_http(verb, url, options = {})
|
83
119
|
response = client.http(verb, url, options)
|
84
120
|
[response.code, response.headers, response.body]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-http'
|
3
|
-
s.version = '1.
|
3
|
+
s.version = '1.2.0'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = 'This filter requests data from a RESTful Web Service.'
|
6
6
|
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-http. This gem is not a stand-alone program'
|
@@ -27,7 +27,9 @@ Gem::Specification.new do |s|
|
|
27
27
|
|
28
28
|
# Gem dependencies
|
29
29
|
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
|
30
|
+
s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
|
30
31
|
s.add_runtime_dependency 'logstash-mixin-http_client', '>= 5.0.0', '< 9.0.0'
|
32
|
+
s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
|
31
33
|
|
32
34
|
s.add_development_dependency 'logstash-devutils'
|
33
35
|
end
|
data/spec/filters/http_spec.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'logstash/devutils/rspec/spec_helper'
|
2
|
+
require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
|
2
3
|
require 'logstash/filters/http'
|
3
4
|
|
4
5
|
describe LogStash::Filters::Http do
|
@@ -6,34 +7,74 @@ describe LogStash::Filters::Http do
|
|
6
7
|
let(:event) { LogStash::Event.new(data) }
|
7
8
|
let(:data) { { "message" => "test" } }
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
10
|
+
let(:url) { 'https://a-non-existent.url1' }
|
11
|
+
let(:config) { { "url" => url, 'target_body' => '[the][body]' } }
|
12
|
+
|
13
|
+
let(:response) { [200, {}, "Bom dia"] }
|
14
|
+
|
15
|
+
describe 'target_body default', :ecs_compatibility_support do
|
16
|
+
|
17
|
+
let(:config) { { "url" => url, "ecs_compatibility" => ecs_compatibility } }
|
18
|
+
|
19
|
+
ecs_compatibility_matrix(:disabled, :v1, :v8) do |ecs_select|
|
20
|
+
|
21
|
+
it "has a default body_target (in legacy mode)" do
|
22
|
+
subject.register
|
23
|
+
allow(subject).to receive(:request_http).and_return(response)
|
24
|
+
subject.filter(event)
|
25
|
+
|
26
|
+
expect(event.get('body')).to eq("Bom dia")
|
27
|
+
end if ecs_select.active_mode == :disabled
|
28
|
+
|
29
|
+
it "fails due missing body_target (in ECS mode)" do
|
30
|
+
expect { subject }.to raise_error(LogStash::ConfigurationError)
|
31
|
+
end if ecs_select.active_mode != :disabled
|
32
|
+
|
14
33
|
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
describe 'response body handling', :ecs_compatibility_support do
|
38
|
+
|
39
|
+
let(:url) { 'http://laceholder.typicode.com/users/10' }
|
40
|
+
|
15
41
|
before(:each) do
|
42
|
+
subject.register
|
43
|
+
|
16
44
|
allow(subject).to receive(:request_http).and_return(response)
|
17
45
|
subject.filter(event)
|
18
46
|
end
|
19
47
|
|
20
|
-
|
21
|
-
|
48
|
+
ecs_compatibility_matrix(:disabled, :v1, :v8) do
|
49
|
+
|
50
|
+
let(:config) { super().merge "ecs_compatibility" => ecs_compatibility, 'target_body' => 'gw-response' }
|
51
|
+
|
52
|
+
context "when body is JSON" do
|
53
|
+
context "and headers are set correctly" do
|
22
54
|
|
23
|
-
|
24
|
-
|
55
|
+
let(:response) { [200, {"content-type" => "application/json"}, "{\"id\": 10}"] }
|
56
|
+
|
57
|
+
it "fetches and writes body to target" do
|
58
|
+
expect(event.get('[gw-response][id]')).to eq(10)
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
25
62
|
end
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
let(:
|
63
|
+
|
64
|
+
context 'with body target' do
|
65
|
+
|
66
|
+
let(:config) { super().merge "target_body" => '[rest]' }
|
30
67
|
|
31
68
|
it "fetches and writes body to target" do
|
32
|
-
expect(event.get('
|
69
|
+
expect(event.get('rest')).to eq("Bom dia")
|
70
|
+
expect(event.include?('body')).to be false
|
33
71
|
end
|
72
|
+
|
34
73
|
end
|
74
|
+
|
35
75
|
end
|
36
76
|
end
|
77
|
+
|
37
78
|
describe 'URL parameter' do
|
38
79
|
before(:each) { subject.register }
|
39
80
|
context "when url contains field references" do
|
@@ -45,6 +86,7 @@ describe LogStash::Filters::Http do
|
|
45
86
|
it "interpolates request url using event data" do
|
46
87
|
expect(subject).to receive(:request_http).with(anything, "http://stringsize.com/test", anything).and_return(response)
|
47
88
|
subject.filter(event)
|
89
|
+
expect(event.get('size')).to eql '4'
|
48
90
|
end
|
49
91
|
end
|
50
92
|
end
|
@@ -68,38 +110,77 @@ describe LogStash::Filters::Http do
|
|
68
110
|
expect(event.get('tags')).to include('_httprequestfailure')
|
69
111
|
end
|
70
112
|
end
|
71
|
-
|
113
|
+
|
114
|
+
describe "headers", :ecs_compatibility_support do
|
72
115
|
before(:each) { subject.register }
|
73
|
-
let(:response)
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
116
|
+
let(:response) do
|
117
|
+
response_headers = {
|
118
|
+
'Server' => 'Apache',
|
119
|
+
'Last-Modified' => 'Mon, 18 Jul 2016 02:36:04 GMT',
|
120
|
+
'X-Backend-Server' => 'logstash.elastic.co',
|
121
|
+
}
|
122
|
+
[200, response_headers, "Bom dia"]
|
123
|
+
end
|
124
|
+
|
125
|
+
let(:url) { "http://stringsize.com" }
|
126
|
+
|
127
|
+
ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
|
128
|
+
|
129
|
+
let(:config) { super().merge "ecs_compatibility" => ecs_compatibility }
|
130
|
+
|
131
|
+
it "sets response headers in the event" do
|
132
|
+
expect(subject).to receive(:request_http).with(anything, config['url'], anything).and_return(response)
|
133
|
+
|
134
|
+
subject.filter(event)
|
135
|
+
|
136
|
+
if ecs_select.active_mode == :disabled
|
137
|
+
expect(event.get('headers')).to include "Server" => "Apache"
|
138
|
+
expect(event.get('headers')).to include "X-Backend-Server" => "logstash.elastic.co"
|
139
|
+
else
|
140
|
+
expect(event.include?('headers')).to be false
|
141
|
+
expect(event.get('[@metadata][filter][http][response][headers]')).to include "Server" => "Apache"
|
142
|
+
expect(event.get('[@metadata][filter][http][response][headers]')).to include "X-Backend-Server" => "logstash.elastic.co"
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
context 'with a headers target' do
|
147
|
+
|
148
|
+
let(:config) { super().merge "target_headers" => '[res][headers]' }
|
149
|
+
|
150
|
+
it "sets response headers in the event" do
|
151
|
+
expect(subject).to receive(:request_http).with(anything, config['url'], anything).and_return(response)
|
152
|
+
|
153
|
+
subject.filter(event)
|
154
|
+
|
155
|
+
expect(event.get('[res][headers]')).to include "Server" => "Apache"
|
156
|
+
expect(event.get('[res][headers]').keys).to include "Last-Modified"
|
157
|
+
end
|
158
|
+
|
82
159
|
end
|
160
|
+
|
161
|
+
end
|
162
|
+
|
163
|
+
context "(extra) request headers" do
|
164
|
+
let(:headers) { { "Cache-Control" => "nocache" } }
|
165
|
+
let(:config) { super().merge "headers" => headers }
|
166
|
+
|
83
167
|
it "are included in the request" do
|
84
168
|
expect(subject).to receive(:request_http) do |verb, url, options|
|
85
|
-
expect(options.fetch(:headers, {})).to include(headers)
|
169
|
+
expect( options.fetch(:headers, {}) ).to include(headers)
|
86
170
|
end.and_return(response)
|
171
|
+
|
87
172
|
subject.filter(event)
|
88
173
|
end
|
89
174
|
end
|
90
175
|
end
|
176
|
+
|
91
177
|
describe "query string parameters" do
|
92
178
|
before(:each) { subject.register }
|
93
179
|
let(:response) { [200, {}, "Bom dia"] }
|
94
180
|
context "when set" do
|
95
181
|
let(:query) { { "color" => "green" } }
|
96
|
-
let(:config)
|
97
|
-
|
98
|
-
"url" => "http://stringsize.com/%{message}",
|
99
|
-
"target_body" => "size",
|
100
|
-
"query" => query
|
101
|
-
}
|
102
|
-
end
|
182
|
+
let(:config) { super().merge "query" => query }
|
183
|
+
|
103
184
|
it "are included in the request" do
|
104
185
|
expect(subject).to receive(:request_http).with(anything, anything, include(:query => query)).and_return(response)
|
105
186
|
subject.filter(event)
|
@@ -109,21 +190,11 @@ describe LogStash::Filters::Http do
|
|
109
190
|
describe "request body" do
|
110
191
|
before(:each) { subject.register }
|
111
192
|
let(:response) { [200, {}, "Bom dia"] }
|
112
|
-
let(:
|
113
|
-
|
114
|
-
"url" => "http://stringsize.com",
|
115
|
-
"body" => body
|
116
|
-
}
|
117
|
-
end
|
193
|
+
let(:url) { "http://stringsize.com" }
|
194
|
+
let(:config) { super().merge "body" => body }
|
118
195
|
|
119
196
|
describe "format" do
|
120
|
-
let(:config)
|
121
|
-
{
|
122
|
-
"url" => "http://stringsize.com",
|
123
|
-
"body_format" => body_format,
|
124
|
-
"body" => body
|
125
|
-
}
|
126
|
-
end
|
197
|
+
let(:config) { super().merge "body_format" => body_format, "body" => body }
|
127
198
|
|
128
199
|
context "when is json" do
|
129
200
|
let(:body_format) { "json" }
|
@@ -190,13 +261,8 @@ describe LogStash::Filters::Http do
|
|
190
261
|
end
|
191
262
|
describe "verb" do
|
192
263
|
let(:response) { [200, {}, "Bom dia"] }
|
193
|
-
let(:config)
|
194
|
-
|
195
|
-
"verb" => verb,
|
196
|
-
"url" => "http://stringsize.com",
|
197
|
-
"target_body" => "size"
|
198
|
-
}
|
199
|
-
end
|
264
|
+
let(:config) { super().merge "verb" => verb }
|
265
|
+
|
200
266
|
["GET", "HEAD", "POST", "DELETE", "PATCH", "PUT"].each do |verb_string|
|
201
267
|
let(:verb) { verb_string }
|
202
268
|
context "when verb #{verb_string} is set" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-http
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-12-
|
11
|
+
date: 2021-12-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -30,6 +30,20 @@ dependencies:
|
|
30
30
|
- - "<="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - "~>"
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '1.2'
|
39
|
+
name: logstash-mixin-ecs_compatibility_support
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '1.2'
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
requirement: !ruby/object:Gem::Requirement
|
35
49
|
requirements:
|
@@ -50,6 +64,20 @@ dependencies:
|
|
50
64
|
- - "<"
|
51
65
|
- !ruby/object:Gem::Version
|
52
66
|
version: 9.0.0
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
requirement: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - "~>"
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '1.0'
|
73
|
+
name: logstash-mixin-validator_support
|
74
|
+
prerelease: false
|
75
|
+
type: :runtime
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - "~>"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '1.0'
|
53
81
|
- !ruby/object:Gem::Dependency
|
54
82
|
requirement: !ruby/object:Gem::Requirement
|
55
83
|
requirements:
|