logstash-input-elasticsearch 4.3.0 → 4.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +15 -0
- data/docs/index.asciidoc +28 -8
- data/lib/logstash/inputs/elasticsearch.rb +86 -8
- data/logstash-input-elasticsearch.gemspec +4 -2
- data/spec/es_helper.rb +59 -0
- data/spec/fixtures/test_certs/ca/ca.crt +32 -0
- data/spec/fixtures/test_certs/ca/ca.key +51 -0
- data/spec/fixtures/test_certs/test.crt +36 -0
- data/spec/fixtures/test_certs/test.key +51 -0
- data/spec/inputs/elasticsearch_spec.rb +90 -57
- data/spec/inputs/integration/elasticsearch_spec.rb +63 -0
- metadata +42 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a5e1d79e9b4b6fdd1deaf7575f6fe3c5c91c578a327478d3dec2172ca3d12cc0
|
4
|
+
data.tar.gz: 046771cceceb57a4a9f0ad395f55fa273dc80b224fa50eab410c3a3b0212ff2b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 24e767e7f20fe2320ad6e103762e0a5158d23d3c63ed781675d1c9fd648bea8fcbc654722bd2c792cedccdc5f534617ac78d552938e60a8b7c957e5b078e1d70
|
7
|
+
data.tar.gz: 9133d597e944199974171430a9c6bbe1acc7c3709d5d6371176e26152d53b0256dc62cac045e614f8123e10b904aa5af9c31a1d6c58be47b83823c0762d95fda
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,18 @@
|
|
1
|
+
## 4.5.0
|
2
|
+
- Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
|
3
|
+
|
4
|
+
## 4.4.0
|
5
|
+
- Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
|
6
|
+
|
7
|
+
## 4.3.3
|
8
|
+
- Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
|
9
|
+
|
10
|
+
## 4.3.2
|
11
|
+
- Fixed broken link to Elasticsearch Reference [#106](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/106)
|
12
|
+
|
13
|
+
## 4.3.1
|
14
|
+
- Fixed deeplink to Elasticsearch Reference [#103](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/103)
|
15
|
+
|
1
16
|
## 4.3.0
|
2
17
|
- Added managed slice scrolling with `slices` option
|
3
18
|
|
data/docs/index.asciidoc
CHANGED
@@ -87,6 +87,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
87
87
|
|=======================================================================
|
88
88
|
|Setting |Input type|Required
|
89
89
|
| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
|
90
|
+
| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
|
91
|
+
| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
|
90
92
|
| <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
|
91
93
|
| <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
|
92
94
|
| <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
|
@@ -113,8 +115,27 @@ input plugins.
|
|
113
115
|
* Value type is <<path,path>>
|
114
116
|
* There is no default value for this setting.
|
115
117
|
|
116
|
-
SSL Certificate Authority file in PEM encoded format, must also
|
117
|
-
|
118
|
+
SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
|
119
|
+
|
120
|
+
[id="plugins-{type}s-{plugin}-cloud_auth"]
|
121
|
+
===== `cloud_auth`
|
122
|
+
|
123
|
+
* Value type is <<password,password>>
|
124
|
+
* There is no default value for this setting.
|
125
|
+
|
126
|
+
Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
|
127
|
+
|
128
|
+
For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
|
129
|
+
|
130
|
+
[id="plugins-{type}s-{plugin}-cloud_id"]
|
131
|
+
===== `cloud_id`
|
132
|
+
|
133
|
+
* Value type is <<string,string>>
|
134
|
+
* There is no default value for this setting.
|
135
|
+
|
136
|
+
Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
|
137
|
+
|
138
|
+
For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
|
118
139
|
|
119
140
|
[id="plugins-{type}s-{plugin}-docinfo"]
|
120
141
|
===== `docinfo`
|
@@ -163,11 +184,10 @@ It will be removed in the next major version of Logstash.
|
|
163
184
|
* Value type is <<array,array>>
|
164
185
|
* Default value is `["_index", "_type", "_id"]`
|
165
186
|
|
166
|
-
If document metadata storage is requested by enabling the `docinfo`
|
167
|
-
option
|
168
|
-
|
169
|
-
|
170
|
-
in the Elasticsearch documentation for more information.
|
187
|
+
If document metadata storage is requested by enabling the `docinfo` option, this
|
188
|
+
option lists the metadata fields to save in the current event. See
|
189
|
+
{ref}/mapping-fields.html[Meta-Fields] in the Elasticsearch documentation for
|
190
|
+
more information.
|
171
191
|
|
172
192
|
[id="plugins-{type}s-{plugin}-docinfo_target"]
|
173
193
|
===== `docinfo_target`
|
@@ -260,7 +280,7 @@ This allows you to set the maximum number of hits returned per scroll.
|
|
260
280
|
|
261
281
|
In some cases, it is possible to improve overall throughput by consuming multiple
|
262
282
|
distinct slices of a query simultaneously using the
|
263
|
-
https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-
|
283
|
+
https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-body.html#sliced-scroll[Sliced Scroll API],
|
264
284
|
especially if the pipeline is spending significant time waiting on Elasticsearch
|
265
285
|
to provide results.
|
266
286
|
|
@@ -2,8 +2,10 @@
|
|
2
2
|
require "logstash/inputs/base"
|
3
3
|
require "logstash/namespace"
|
4
4
|
require "logstash/json"
|
5
|
+
require "logstash/util/safe_uri"
|
5
6
|
require "base64"
|
6
7
|
|
8
|
+
|
7
9
|
# .Compatibility Note
|
8
10
|
# [NOTE]
|
9
11
|
# ================================================================================
|
@@ -68,6 +70,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
68
70
|
# Port defaults to 9200
|
69
71
|
config :hosts, :validate => :array
|
70
72
|
|
73
|
+
# Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
|
74
|
+
#
|
75
|
+
# For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
|
76
|
+
config :cloud_id, :validate => :string
|
77
|
+
|
71
78
|
# The index or alias to search.
|
72
79
|
config :index, :validate => :string, :default => "logstash-*"
|
73
80
|
|
@@ -133,6 +140,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
133
140
|
# Basic Auth - password
|
134
141
|
config :password, :validate => :password
|
135
142
|
|
143
|
+
# Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
|
144
|
+
#
|
145
|
+
# For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
|
146
|
+
config :cloud_auth, :validate => :password
|
147
|
+
|
136
148
|
# SSL
|
137
149
|
config :ssl, :validate => :boolean, :default => false
|
138
150
|
|
@@ -149,6 +161,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
149
161
|
def register
|
150
162
|
require "elasticsearch"
|
151
163
|
require "rufus/scheduler"
|
164
|
+
require "elasticsearch/transport/transport/http/manticore"
|
152
165
|
|
153
166
|
@options = {
|
154
167
|
:index => @index,
|
@@ -163,12 +176,17 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
163
176
|
|
164
177
|
transport_options = {}
|
165
178
|
|
179
|
+
fill_user_password_from_cloud_auth
|
180
|
+
|
166
181
|
if @user && @password
|
167
182
|
token = Base64.strict_encode64("#{@user}:#{@password.value}")
|
168
183
|
transport_options[:headers] = { :Authorization => "Basic #{token}" }
|
169
184
|
end
|
170
185
|
|
171
|
-
|
186
|
+
fill_hosts_from_cloud_id
|
187
|
+
@hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
|
188
|
+
|
189
|
+
hosts = if @ssl
|
172
190
|
@hosts.map do |h|
|
173
191
|
host, port = h.split(":")
|
174
192
|
{ :host => host, :scheme => 'https', :port => port }
|
@@ -176,12 +194,12 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
176
194
|
else
|
177
195
|
@hosts
|
178
196
|
end
|
197
|
+
ssl_options = { :ssl => true, :ca_file => @ca_file } if @ssl && @ca_file
|
198
|
+
ssl_options ||= {}
|
179
199
|
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
@client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
|
200
|
+
@client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options,
|
201
|
+
:transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore,
|
202
|
+
:ssl => ssl_options)
|
185
203
|
end
|
186
204
|
|
187
205
|
|
@@ -273,10 +291,70 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
273
291
|
end
|
274
292
|
|
275
293
|
def scroll_request scroll_id
|
276
|
-
|
294
|
+
client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
|
277
295
|
end
|
278
296
|
|
279
297
|
def search_request(options)
|
280
|
-
|
298
|
+
client.search(options)
|
299
|
+
end
|
300
|
+
|
301
|
+
attr_reader :client
|
302
|
+
|
303
|
+
def hosts_default?(hosts)
|
304
|
+
hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
|
305
|
+
end
|
306
|
+
|
307
|
+
def fill_hosts_from_cloud_id
|
308
|
+
return unless @cloud_id
|
309
|
+
|
310
|
+
if @hosts && !hosts_default?(@hosts)
|
311
|
+
raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
|
312
|
+
end
|
313
|
+
@hosts = parse_host_uri_from_cloud_id(@cloud_id)
|
281
314
|
end
|
315
|
+
|
316
|
+
def fill_user_password_from_cloud_auth
|
317
|
+
return unless @cloud_auth
|
318
|
+
|
319
|
+
if @user || @password
|
320
|
+
raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
|
321
|
+
end
|
322
|
+
@user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
|
323
|
+
params['user'], params['password'] = @user, @password
|
324
|
+
end
|
325
|
+
|
326
|
+
def parse_host_uri_from_cloud_id(cloud_id)
|
327
|
+
begin # might not be available on older LS
|
328
|
+
require 'logstash/util/cloud_setting_id'
|
329
|
+
rescue LoadError
|
330
|
+
raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
|
331
|
+
'please upgrade your installation (or set hosts instead).'
|
332
|
+
end
|
333
|
+
|
334
|
+
begin
|
335
|
+
cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
|
336
|
+
rescue ArgumentError => e
|
337
|
+
raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
|
338
|
+
end
|
339
|
+
cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
|
340
|
+
LogStash::Util::SafeURI.new(cloud_uri)
|
341
|
+
end
|
342
|
+
|
343
|
+
def parse_user_password_from_cloud_auth(cloud_auth)
|
344
|
+
begin # might not be available on older LS
|
345
|
+
require 'logstash/util/cloud_setting_auth'
|
346
|
+
rescue LoadError
|
347
|
+
raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
|
348
|
+
'please upgrade your installation (or set user/password instead).'
|
349
|
+
end
|
350
|
+
|
351
|
+
cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
|
352
|
+
begin
|
353
|
+
cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
|
354
|
+
rescue ArgumentError => e
|
355
|
+
raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
|
356
|
+
end
|
357
|
+
[ cloud_auth.username, cloud_auth.password ]
|
358
|
+
end
|
359
|
+
|
282
360
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.
|
4
|
+
s.version = '4.5.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
24
|
|
25
|
-
s.add_runtime_dependency 'elasticsearch',
|
25
|
+
s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'
|
26
26
|
|
27
27
|
s.add_runtime_dependency 'logstash-codec-json'
|
28
28
|
s.add_runtime_dependency 'logstash-codec-plain'
|
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
|
|
30
30
|
s.add_runtime_dependency 'tzinfo'
|
31
31
|
s.add_runtime_dependency 'tzinfo-data'
|
32
32
|
s.add_runtime_dependency 'rufus-scheduler'
|
33
|
+
s.add_runtime_dependency 'manticore', "~> 0.6"
|
34
|
+
s.add_runtime_dependency 'faraday', "~> 0.15.4"
|
33
35
|
|
34
36
|
s.add_development_dependency 'logstash-devutils'
|
35
37
|
s.add_development_dependency 'timecop'
|
data/spec/es_helper.rb
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
module ESHelper
|
2
|
+
def self.get_host_port
|
3
|
+
return "elasticsearch:9200" if ENV["INTEGRATION"] == "true" || ENV["SECURE_INTEGRATION"] == "true"
|
4
|
+
raise "This setting is only used for integration tests"
|
5
|
+
end
|
6
|
+
|
7
|
+
def self.get_client(options = {})
|
8
|
+
ssl_options = {}
|
9
|
+
hosts = [get_host_port]
|
10
|
+
|
11
|
+
if options[:ca_file]
|
12
|
+
ssl_options = { :ssl => true, :ca_file => options[:ca_file] }
|
13
|
+
hosts.map! do |h|
|
14
|
+
host, port = h.split(":")
|
15
|
+
{ :host => host, :scheme => 'https', :port => port }
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
transport_options = {}
|
20
|
+
|
21
|
+
if options[:user] && options[:password]
|
22
|
+
token = Base64.strict_encode64("#{options[:user]}:#{options[:password]}")
|
23
|
+
transport_options[:headers] = { :Authorization => "Basic #{token}" }
|
24
|
+
end
|
25
|
+
|
26
|
+
@client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options, :ssl => ssl_options,
|
27
|
+
:transport_class => ::Elasticsearch::Transport::Transport::HTTP::Manticore)
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.doc_type
|
31
|
+
if ESHelper.es_version_satisfies?(">=8")
|
32
|
+
nil
|
33
|
+
elsif ESHelper.es_version_satisfies?(">=7")
|
34
|
+
"_doc"
|
35
|
+
else
|
36
|
+
"doc"
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.index_doc(es, params)
|
41
|
+
type = doc_type
|
42
|
+
params[:type] = doc_type unless type.nil?
|
43
|
+
es.index(params)
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.es_version
|
47
|
+
ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.es_version_satisfies?(*requirement)
|
51
|
+
es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
|
52
|
+
if es_version.nil?
|
53
|
+
puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
|
54
|
+
return false
|
55
|
+
end
|
56
|
+
es_release_version = Gem::Version.new(es_version).release
|
57
|
+
Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIFeTCCA2GgAwIBAgIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDQYJKoZIhvcNAQEL
|
3
|
+
BQAwTDELMAkGA1UEBhMCUFQxCzAJBgNVBAgMAk5BMQ8wDQYDVQQHDAZMaXNib24x
|
4
|
+
DjAMBgNVBAoMBU15TGFiMQ8wDQYDVQQDDAZSb290Q0EwHhcNMTkwNzE1MTMxMTI5
|
5
|
+
WhcNMjQwNzE0MTMxMTI5WjBMMQswCQYDVQQGEwJQVDELMAkGA1UECAwCTkExDzAN
|
6
|
+
BgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMMBlJvb3RDQTCC
|
7
|
+
AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAMtTMqAWuH17b9XqPa5L3HNq
|
8
|
+
gnZ958+gvcOt7Q/sOEvcDQJgkzZ+Gywh5er5JF2iomYOHiD5JncYr4YmRQKuYfD6
|
9
|
+
B1WI5FuQthD/OlA1/RHqtbY27J33SaO66ro8gal7vjHrXKQkefVYRwdfO6DqqbhV
|
10
|
+
6L4sMiy8FzQ55TMpoM35cWuvoAMxvSQqGZ4pYYKnfNSGhzHvssfNS1xu/Lwb7Vju
|
11
|
+
4jPhp+43BkGwEimI5km7jNC1nwjiHtxDsY/s93AKa/vLktXKUK5nA3jjJOhAbRTV
|
12
|
+
nbOAgxFt0YbX98xW/aUqscgBUVs9J/MyTRMwVKJ7Vsmth1PdJQksUASuzESlSPl0
|
13
|
+
9dMjTQ+MXzJDt0JvX8SIJPmbBng78MSaCUhpOZiii1l2mBfPWejx20I/SMCUNmzb
|
14
|
+
wm2w9JD50Jv2iX4l4ge4H1CIK1/orW1pdY9xPL0uKYm6ADsDC0B8sGgNMBXeB6aL
|
15
|
+
ojY1/ITwmmfpfk9c/yWPfC7stHgCYRAv5MfGAsmv0/ya5VrWQGBJkFiYy1pon6nx
|
16
|
+
UjCbgn0RABojRoGdhhY3QDipgwmSgFZxr064RFr1bt/Ml3MJmPf535mSwPdk/j/z
|
17
|
+
w4IZTvlmwKW3FyMDhwYL/zX7J0c6MzMPLEdi73Qjzmr3ENIrir4O86wNz81YRfYk
|
18
|
+
g9ZX8yKJK9LBAUrYCjJ3AgMBAAGjUzBRMB0GA1UdDgQWBBShWnSceOrqYn9Qa4WG
|
19
|
+
dIrvKNs/KzAfBgNVHSMEGDAWgBShWnSceOrqYn9Qa4WGdIrvKNs/KzAPBgNVHRMB
|
20
|
+
Af8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBRQK0m3t5h2Y3CUCJYLMiCUge4
|
21
|
+
UOzvpCoawSXH1FP2ycA+P1bP8H8htjwvV334ZADlQrDQRu0hqa1T+DxwhLxNOxgE
|
22
|
+
1XCthN3TTyd3O1mT4NmT6mcn2wYSn/JC6fPwFcloX8BcUvxl+xwmOgL/pzgf1ekK
|
23
|
+
MVS0n+r3bzdFTgGnvsmxmPHe2bUhyXXqzQIx3ObSGtuKYUu7aZEysEtJhaR+vGTd
|
24
|
+
jjTOV2S71edVlKTxRLZpHgoTZpBL/phwRQ63vdef4ftNGs0glGDc0yqXGMxMALOl
|
25
|
+
Up7+H4HI99rldZcul6oZ+ORltt047Hk7ctWb20SqxEH9tGLXKm6hDEL9HzyFXeyJ
|
26
|
+
DAue1GF+3H0KvsjSs5XH7LHMuJDCuSP64+h9gzkI+q06oBNX/9pQyQaHj0K4don8
|
27
|
+
lWOMLI4gQibV7R1Opt2feA8MwWxouP/yni8IX6sPePVQ+fLEk1C+Kg+x6k1yQHEM
|
28
|
+
36BEP6iYOYvqG0OIjMas2U7Yhn2wWlVm9It3WMyaW8ZPI8kwc3dx715dZuNg/zjd
|
29
|
+
rJS678BNBVxInc7dzpY6el0Lr70CGwiJpX/N9P1yiTFZ7GZm3Kax8QnTtvqXzRIy
|
30
|
+
sBgt8BVZHUe1lWFYlG+jlakiXqz752nmHuwif7iBI4iWzRmW2vYPfTEmYPRLZES2
|
31
|
+
nIg9fQPvVw+fIHACZQ==
|
32
|
+
-----END CERTIFICATE-----
|
@@ -0,0 +1,51 @@
|
|
1
|
+
-----BEGIN RSA PRIVATE KEY-----
|
2
|
+
MIIJKAIBAAKCAgEAy1MyoBa4fXtv1eo9rkvcc2qCdn3nz6C9w63tD+w4S9wNAmCT
|
3
|
+
Nn4bLCHl6vkkXaKiZg4eIPkmdxivhiZFAq5h8PoHVYjkW5C2EP86UDX9Eeq1tjbs
|
4
|
+
nfdJo7rqujyBqXu+MetcpCR59VhHB187oOqpuFXoviwyLLwXNDnlMymgzflxa6+g
|
5
|
+
AzG9JCoZnilhgqd81IaHMe+yx81LXG78vBvtWO7iM+Gn7jcGQbASKYjmSbuM0LWf
|
6
|
+
COIe3EOxj+z3cApr+8uS1cpQrmcDeOMk6EBtFNWds4CDEW3Rhtf3zFb9pSqxyAFR
|
7
|
+
Wz0n8zJNEzBUontWya2HU90lCSxQBK7MRKVI+XT10yNND4xfMkO3Qm9fxIgk+ZsG
|
8
|
+
eDvwxJoJSGk5mKKLWXaYF89Z6PHbQj9IwJQ2bNvCbbD0kPnQm/aJfiXiB7gfUIgr
|
9
|
+
X+itbWl1j3E8vS4piboAOwMLQHywaA0wFd4HpouiNjX8hPCaZ+l+T1z/JY98Luy0
|
10
|
+
eAJhEC/kx8YCya/T/JrlWtZAYEmQWJjLWmifqfFSMJuCfREAGiNGgZ2GFjdAOKmD
|
11
|
+
CZKAVnGvTrhEWvVu38yXcwmY9/nfmZLA92T+P/PDghlO+WbApbcXIwOHBgv/Nfsn
|
12
|
+
RzozMw8sR2LvdCPOavcQ0iuKvg7zrA3PzVhF9iSD1lfzIokr0sEBStgKMncCAwEA
|
13
|
+
AQKCAgBVVGU6qk5i2xrkO5oHO+8YwOpfsBdJG7yIEsYamJhHveH3zW/6vpHIw7Eq
|
14
|
+
G8UXRtnA2svqKqXp9YI0Wns71NNlvoi1bO3pP6IpH/PpFb9PdaEtB3/mC5HsFNXN
|
15
|
+
svb3mecILC6E9In6XUHx5hWwQstXgTZcGVA1VfqnAGUgQ6goyTbAasRMkuM9+i0m
|
16
|
+
I1e47XGF/69dVatCDvZBpJKMn2vMlvR3sYw4fP8zMiFtLPb4mq1OKerEX6Fz7zTl
|
17
|
+
oh119+m5koXdEzso9jKO2UTz85XT2JKGcriO5/e3D4v/RcLNPk2+Ek+CavgJKGMQ
|
18
|
+
WogqaHjTyu+wUm7omqA6VuGDLZqh1r0xYR+EXVMAudLjy7/NtAaE4MVOqVRs4WVd
|
19
|
+
sGccyirkTosxlvK3/vTfsp0VQtreBbxO1maqR5od0aa36MrP4Sk5O07yB9GAthp8
|
20
|
+
5qlqtiYaO2Hcq2KJjKPUGwXlAWFZtENQe+G/jy+gYVDwKRInK7f7HubZlAMwsq2S
|
21
|
+
LSjtgvhqayAMsa7HoeevSVPLVdFb1IVkIw2jgMhXRgxmKa8WzbAUs124f9ey9z81
|
22
|
+
si7w+qpZHq9LGChBjweTbd0abCianyRGHZIlDBE43XEcs3easxuHM6eOoJz0B7aj
|
23
|
+
oCXBCo/6Zd0om4ll5jva2+VOH2wTkZk7OhPiGU2f4g7kTJNAAQKCAQEA7YT3UdjN
|
24
|
+
HybAD3c/a5Kh17R4zCvymQkUWBs80ZaV9LlCYgie6aWlY6e+9m6vpDhY8dJZd+Zm
|
25
|
+
hlAF3VitRLw3SQUEImEC1caS1q99o1eQxMGu+mk9OiibF9PzZwoPE6zt5EZ0I/Ha
|
26
|
+
ifmf0Jn3xLyWF4qOKvO3gbWLknirDKffzNnWtFr4SQlEwtOR4m7IFDEz7e7RoGlv
|
27
|
+
K1qEFyK1PCfR8GeVHXWkJ3udHJRIZlEtNNLkHzU4nCRRYTvQ4l67rD9Tj7CoLbH1
|
28
|
+
2OGSSvAkg+1lTBBs6RXZvhVvLrJVtQTXR7Oi8Z3mi3iJu9oWYa/OFaJl4lAN9xTe
|
29
|
+
QY0u0J1+AS5qAQKCAQEA2yUhO3rC1A7qHgbY4dAsx8f8yy9D0yCrI9OLnPQNF3ws
|
30
|
+
4mC1fUS60+07u0FYkgU3zIDwdLj5nsxWjB4ciY4hCgwz7pNJWlowurrfTvQNlqvC
|
31
|
+
m+Jrt1HYoaV+73mSj+rGv89HXWBW2I/1ED37BRoNB/YIMd/MUL8h0ubt3LIVaRow
|
32
|
+
41DT3dM969zuw3Avpx1uXQdnijJ1kA3oHpJ756YLHri/Nv6K0hJmGAbMrHPRxuhY
|
33
|
+
hYrxPJPVlp5mWUIjNkKoaNl3du3a6iVSbf7W15LxhAHmkKozxnhqoMldI6C8R548
|
34
|
+
IKGyW4wo3GQvcEGPhgGnz2lswmvtx/6cWMv81b7sdwKCAQAXiC3sqPshk/hBUAIz
|
35
|
+
iTKJqXKyX8RITzL5y7EJ3s56kYQ3LD16TpQFPJBZ3/t83KxLQRjmHSiZNAJwvKFm
|
36
|
+
BvO/Q0T2lv/t6B+SL47WCQ3pwHqyioyrX2yGPtSFp+R4gZCMJkLJcOPC+b1QsIBw
|
37
|
+
uHJyYKLXNJBAxJjQaS4hMdylcguokL66lbV/S/DPK0SdY4aOkzOnneqKtAwUMrcb
|
38
|
+
/6H4HHsUkRwzYTbepv5JTM+axS4evWofZiW96Ww3kUUsupVvzgPLiy8dTrPswsAL
|
39
|
+
ZhC8KYBw015gS8VZLgf5yEH/85c4MvmtZcLXnrLK+N0FHbLUajQH/8RJYFB8EK50
|
40
|
+
NYIBAoIBAQCNO8/AIqz/uCEAew858U168BOm62492lcRvtvCqrLpSNkwiH1PH4V8
|
41
|
+
4e7WDxZC/WPpw8u0niYaRr0cjqd7q4L1k8nAX2It/eRb4+XZX4aGbWn9xx3+xpvk
|
42
|
+
CeHV+rcPU5MFuVPYBSfTaGvbLObjcdemItVDN2XZQGVPJA92ZbtQwlVxmv0AgIzu
|
43
|
+
vrOOd3XusRQYlpYPRdfooF3RnjyGncea5BwwGDpliwALSg6MshQnqkSqxFIW5XwJ
|
44
|
+
F0sFCY/h/3HEKStKFZ85uhX21/+kbYqDtinfYCIALkkiGMSTGptdWMiNi0iEob8P
|
45
|
+
0u2T3lzeU6DQFrTGVIKpmxkuTUFEjEifAoIBAH4nyu4ei4u7sGUhSZ79egUbqBZR
|
46
|
+
pjYblM8NB5UOAVmmbaswCWozsnsaBdIgymeokQXDPqIOwadP8IgGrgIxS5phdgvE
|
47
|
+
CNepxQDoVTXYEecPjc0LL4Kb+urmJL7HEP9BteIkc+7l8b9USDhNlJeCXICoJKBu
|
48
|
+
bNxgm60ZuoObx7h5APq9wC4x8Xj7AxQKu57Ied/tRFPCHW4UbhZhJfrnS2xTTk0u
|
49
|
+
z71AS7akI/NPfm3nLviISZeDzTgYs6vLYr/j4JUlcw1z6UpX4DvNm/MULi7ItXP5
|
50
|
+
yV2H8jpCdjAe+OoC3OHTuQ8FQR29y758HUY7iF8ruuqUSWxF7pfL/a27EMw=
|
51
|
+
-----END RSA PRIVATE KEY-----
|
@@ -0,0 +1,36 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIGQjCCBCqgAwIBAgIBAzANBgkqhkiG9w0BAQsFADBMMQswCQYDVQQGEwJQVDEL
|
3
|
+
MAkGA1UECAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzAN
|
4
|
+
BgNVBAMMBlJvb3RDQTAeFw0xOTA3MTUxMzEzMDVaFw0yMjA0MTAxMzEzMDVaMFMx
|
5
|
+
CzAJBgNVBAYTAlBUMQswCQYDVQQIDAJOQTEPMA0GA1UEBwwGTGlzYm9uMQ4wDAYD
|
6
|
+
VQQKDAVNeUxhYjEWMBQGA1UEAwwNZWxhc3RpY3NlYXJjaDCCAiIwDQYJKoZIhvcN
|
7
|
+
AQEBBQADggIPADCCAgoCggIBAMYhP2zPOE3ke9naeK+cIPNV91htuoGGARs+mlY/
|
8
|
+
IVxXSvau2ZZ94rkQR2xNL8TLijBNx46mU+kCniy8X5r+LX9seGqdBhhTh/tCJzh8
|
9
|
+
MCzMt2JIijSjVyw28iiCb8/669LMTp5lFlRKajj11jlIpIm3o+OHqUzYwcSOw8og
|
10
|
+
p0A3nvAQ33Srghm/oAcT2umGrFyYXWT6PnGaEJRLUQn7LuHJnRLseCF2Cn/RzFK7
|
11
|
+
/tiVVjImmQiVB3dE9fMR/pVJiO2v0COnWuG+/brXWrQIHk0AuD8pHc6Iw9iZODkc
|
12
|
+
Ao53B41qbvqcbdXFN5XfL4tb+lkBuLioCX7j9zR44awvuj9hKfuqFOFTUBZL2RjV
|
13
|
+
bFMKspGHnytQZF+a+mc5H33G9HiPP3jZE2JjrWlOay+j6ImylMgjcZmHAgaUe3ET
|
14
|
+
1GfnSVZBwO4MMd85taHNvitLnkEREjANSoPUuAJF3SKRHE9K8jUAzhyXflvgNNoM
|
15
|
+
tyczoQ5/L5BNiyA2h+1TU8jWicNDtl1+CtOsgEVBBHA6p/IHhsHbNZWPrYtIO9mh
|
16
|
+
hiJw1R5yrITXnjZY0rObITwyt/e6Sc3YnoQfsSGaLJEG0aDc0RALAhgzj+RY8086
|
17
|
+
2RKOyfdw1sw1RmJKdCf+dOzhPyDpvauvCxrL8UZQTzcBs+qpxOWnZFRWeNsLwoDn
|
18
|
+
6JXXAgMBAAGjggEmMIIBIjAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIGQDAz
|
19
|
+
BglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgU2VydmVyIENlcnRpZmlj
|
20
|
+
YXRlMB0GA1UdDgQWBBRvvz0yGw6Tz2UxbBLAGyzVMtcMUDCBiAYDVR0jBIGAMH6A
|
21
|
+
FKFadJx46upif1BrhYZ0iu8o2z8roVCkTjBMMQswCQYDVQQGEwJQVDELMAkGA1UE
|
22
|
+
CAwCTkExDzANBgNVBAcMBkxpc2JvbjEOMAwGA1UECgwFTXlMYWIxDzANBgNVBAMM
|
23
|
+
BlJvb3RDQYIUU+VHJ91JsLLA1GJYC+UchNfw3hEwDgYDVR0PAQH/BAQDAgWgMBMG
|
24
|
+
A1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQCaABHQxm6mtrM9
|
25
|
+
f7kbgzuhEc47Q+bgrbjxeoIVOeO2Zshdw0SZlfkWvWe0622WSeWMsTBJ3hoaQwZe
|
26
|
+
9FUf1lnsWe6u6oOckiG9OjE0TyXJ7+eghdL1HPeXgJ+4ihwJsRtkNEljWf4HS7/n
|
27
|
+
y5LaFhcXdn2ZdbUKJ7z7zXqzh2Cp8VUBtsR+/IdiLjSN81dQou77/a2M/a/7BI2Z
|
28
|
+
HhUlUx1T7jHzNllJBRF3IaOk72yjoU4cL0qVy9874SXPwdpeFHtvS4TdQTLqnAGR
|
29
|
+
liHJcB1ZNz1sVOXndw3Wbvv6iB5y+IX/Y/kRSHS6zpZGdAb7ar/Vgl+Uvs3fKi44
|
30
|
+
y9hq2b49bYlcSQMtmlimCBDiu82z0aYtVFLalZ2L/W7CMaeE3jpyzu/bbygRv/Bp
|
31
|
+
lKSaUtaFIVgiuRBPwIBDMyai3CJ5L+dJrJPU2JzzQvtJGFQCFCIHd9rqweubZB6V
|
32
|
+
re5cUn4dxlxA5SkZ0amFFV5DpP0YhThA/gq0t/NeWRmCEEBWNXZaqFmDhiYS5mnu
|
33
|
+
Z+NUtv8E332S46RdfneHe961SlMXEFC96I+1HOjXHdXlqKfOU8Qvy8VzsnpjuNE5
|
34
|
+
VTrvnAM1L3LwqtYQYfUWUHYZFYdvh8layA2ImNE7yx/9wIIkw/L1j9m71Upi6WKR
|
35
|
+
FKbYFqzgpWksa+zZ2RYYplUAxq0wYw==
|
36
|
+
-----END CERTIFICATE-----
|
@@ -0,0 +1,51 @@
|
|
1
|
+
-----BEGIN RSA PRIVATE KEY-----
|
2
|
+
MIIJKQIBAAKCAgEAxiE/bM84TeR72dp4r5wg81X3WG26gYYBGz6aVj8hXFdK9q7Z
|
3
|
+
ln3iuRBHbE0vxMuKME3HjqZT6QKeLLxfmv4tf2x4ap0GGFOH+0InOHwwLMy3YkiK
|
4
|
+
NKNXLDbyKIJvz/rr0sxOnmUWVEpqOPXWOUikibej44epTNjBxI7DyiCnQDee8BDf
|
5
|
+
dKuCGb+gBxPa6YasXJhdZPo+cZoQlEtRCfsu4cmdEux4IXYKf9HMUrv+2JVWMiaZ
|
6
|
+
CJUHd0T18xH+lUmI7a/QI6da4b79utdatAgeTQC4PykdzojD2Jk4ORwCjncHjWpu
|
7
|
+
+pxt1cU3ld8vi1v6WQG4uKgJfuP3NHjhrC+6P2Ep+6oU4VNQFkvZGNVsUwqykYef
|
8
|
+
K1BkX5r6Zzkffcb0eI8/eNkTYmOtaU5rL6PoibKUyCNxmYcCBpR7cRPUZ+dJVkHA
|
9
|
+
7gwx3zm1oc2+K0ueQRESMA1Kg9S4AkXdIpEcT0ryNQDOHJd+W+A02gy3JzOhDn8v
|
10
|
+
kE2LIDaH7VNTyNaJw0O2XX4K06yARUEEcDqn8geGwds1lY+ti0g72aGGInDVHnKs
|
11
|
+
hNeeNljSs5shPDK397pJzdiehB+xIZoskQbRoNzREAsCGDOP5FjzTzrZEo7J93DW
|
12
|
+
zDVGYkp0J/507OE/IOm9q68LGsvxRlBPNwGz6qnE5adkVFZ42wvCgOfoldcCAwEA
|
13
|
+
AQKCAgA1FkOATCWx+T6WKMudgh/yE16q+vu2KMmzGxsPcOrnaxxS7JawlBpjq9D3
|
14
|
+
W9coy8DDIJQPzNE+5cyr/+0+Akz+j3nUVy6C5h7RW/BWWjAuUMvyMa2WXQ3GcxJ/
|
15
|
+
eDOtbnYxjTyjhEJvY2EC0hwMTUKJBAONu5PJW2rP19DuH8Gwmzai7GJzSGEbtRST
|
16
|
+
0OYfHE6ioNCldce1eKpokaWtHvh41ySXJXUqwg4eIYC1ylmGfr0RwvXOLuBJPNkJ
|
17
|
+
wBCOv51I0oragsT/J8Wkgn9zLZmw2DiF8+ZgqJSRPLyr0K1+rrX/Vj1WOQPU+3rh
|
18
|
+
VWPP211A7A0qrRuePEbIcHtHP6KPUCepABL44K33zyyOydmnJ7vg3dsW7AN7+Y6O
|
19
|
+
H4B24d1ogn4TJwzpZCfRvqJJVu2wsnzleng9PcpXyHhldB6S9h2fPpNqDUBvfxMv
|
20
|
+
w/fGZ2ZpOeUKRfQ7VVR3XIWwFq/eDhzLicHipaoM+6gKeOZdJPAc0Ew5jvOXQSBD
|
21
|
+
CYCM12a8gnEYd55NLo/fF3wX6Wdq/X6EbWW97gwtmmEqnhcZMxLdeMuKyli22JyX
|
22
|
+
Ik7QIDsmPSWhCkI2JvQ+CAZZp6oMEKuSb7UqqfACQreIuxCUmPTZq/pAEUGSCZGP
|
23
|
+
wnWqOk5jwxJ4d5TQm7g2RgPC6lTd7as1m4+JB8H1cNVpS2d0AQKCAQEA5tL9WjIK
|
24
|
+
u6x1h4a4KPmOQ9B34GxmGM+P9/bQkkJsWYf5eG1MlYPAigAiN0PMYPHDweeROsOO
|
25
|
+
rvmJiWXtmPMVFFSalHVotN6LMj400HhHObg7dADDYTBBGX6QuljxBX9jgUiKSOKO
|
26
|
+
66ngXEyRdUmTAbral3UuSCFGcqG8Khd3taULO2q5JQLEioFT+Lr0UzHywVSJq06O
|
27
|
+
k37aC3zpiQj4S/bJG4GOadYDIENq+gRCIU7Hn4pS1qtxLYMyhtNDzK0pVriYNj9T
|
28
|
+
ydHUxSKZO6ogM4423wVKnKOa2Cj4rgKBDHGpJJ9R0ZhrTbAQOa8LgDy1P6aMlvH/
|
29
|
+
t9bG/HClmuHrFwKCAQEA271gZpIeCFG/XYIbxO8Uh9wG2cHrt7idw0iFTW4PpOBk
|
30
|
+
CGExq7WlkR29BkzxTDEXz5bSgDa8Q83cDf9EjSHVJYVGSYnodz7ZV8jZd2PUdCV1
|
31
|
+
dL1kHV7vqz/KKxuCp7icCwzG9rQ1CjsTv8gBM3cN6DrZgw/2F+HjQpCkeyxc6KBR
|
32
|
+
Q+167yaLvOv3W0BHdSywtiNDU48oSSBpEK2anh7ShjG3BaEr/gAqGsTvxjsl7zDg
|
33
|
+
6MZFegeYPFIEH4ftvLZugPgd3NBg00CfsNRkjVWsH51y4gBy2ZL8d31Q2D2FI94s
|
34
|
+
he57Trvs8t8Y9QHGTMCuUk9IwRBpgapmW+c6G65jQQKCAQB0IPquAjc8LIwE31aP
|
35
|
+
5t4YaC2nj2HeYa8BBf/xVdHFSY/Ncs/w+RRJWb/dJhMRQxFF4QdEFVcX2RvFMK7V
|
36
|
+
IJceX2JWBqvjM4O8h2dy6tCKzZG7zxZ9MxXNicuutUith2W8iY5pFPLqxdDHHw6f
|
37
|
+
f6CiYivPv3jFeVhEG/LbCmuDy8FW5162rCnNtTtWDFkX8i077xhEQ4Wf11ZEKVgl
|
38
|
+
RYoGTeboG8pWnQF9ne2YU8Qhlc0BC0qaDi8mwrcM9gVKWGRP6RdLU5kIFLWDaODH
|
39
|
+
D9Sbm5UnpzXekME6t4JFqaTbaeO7NRyo4pI5x7aiDtsubVyS5WweFSqeh0QdhV8M
|
40
|
+
CVWJAoIBAQCJ7OSFfVG8hxNG7lPf2PFaFZF3PXFncEoWOX7yixLmurIPUHKNBQdX
|
41
|
+
fwMW4VTULjxz4IdgUvr41E47uu6cZ5ASbQUhQ57WumxR3ZAikFbaUGjZTcd5aA2n
|
42
|
+
v/J1/F6WSBoqFWsNjy97rHlI95FJbIEeM1I0IeCmPPMY5RFY/w+SNI7NxFJVqiwr
|
43
|
+
+TDZ5g70TYjdymSIHmN7AwzvmwhiF5atBKeMsQ2b8R58jwCxvI6jBFsnwMv7PKkh
|
44
|
+
s5lC8V8YBKp36UVVRLaB4x5ZL/etfwj7Dyj9EqsERm6R0ebc1ECtstbfekGLugmQ
|
45
|
+
qNhRcTu3EXpZz8oq5NJUwVBef1TJ6zwBAoIBAQC7Oq5AdmLzYOX0AYUuT3Cjgl0u
|
46
|
+
3Tq1c1uqlVaxQGjA3oqua4SR0+kvmRZbZGLpLAVnnwOjUEfUFBBYgP/5Mo/OiCkQ
|
47
|
+
C8eWkSQKXy6OFy5wh4mbL5oJttKjM4ZoB0gpF31+tGOmrfJwacqEPnyZLKzkgBdG
|
48
|
+
djVVKP+HH4XUdB2VXst8tvcif+VTrUsD1nuhGMOgbiHZkdx3eug05wfhnYvWljA+
|
49
|
+
r/4xcq7TmKSoJqkb0OcOkhqjeZWleA6xRtEvHPGRzbEM67FVsVTxr/N9BX5tS9zu
|
50
|
+
YLCNI3tTNsDV0Ac+4rl46rghQ/n2TNSEpwvA/pjytsdPXLOpoapWirrsEiXf
|
51
|
+
-----END RSA PRIVATE KEY-----
|
@@ -125,7 +125,7 @@ describe LogStash::Inputs::Elasticsearch do
|
|
125
125
|
end
|
126
126
|
|
127
127
|
context 'without slices directive' do
|
128
|
-
let(:config) { super
|
128
|
+
let(:config) { super.tap { |h| h.delete('slices') } }
|
129
129
|
it 'runs just one slice' do
|
130
130
|
expect(plugin).to receive(:do_run_slice).with(duck_type(:<<))
|
131
131
|
expect(Thread).to_not receive(:new)
|
@@ -375,15 +375,13 @@ describe LogStash::Inputs::Elasticsearch do
|
|
375
375
|
end
|
376
376
|
|
377
377
|
it 'merges the values if the `docinfo_target` already exist in the `_source` document' do
|
378
|
-
metadata_field = 'metadata_with_hash'
|
379
|
-
|
380
378
|
config_metadata_with_hash = %Q[
|
381
379
|
input {
|
382
380
|
elasticsearch {
|
383
381
|
hosts => ["localhost"]
|
384
382
|
query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
|
385
383
|
docinfo => true
|
386
|
-
docinfo_target => '
|
384
|
+
docinfo_target => 'metadata_with_hash'
|
387
385
|
}
|
388
386
|
}
|
389
387
|
]
|
@@ -392,33 +390,23 @@ describe LogStash::Inputs::Elasticsearch do
|
|
392
390
|
queue.pop
|
393
391
|
end
|
394
392
|
|
395
|
-
expect(event.get("[
|
396
|
-
expect(event.get("[
|
397
|
-
expect(event.get("[
|
398
|
-
expect(event.get("[
|
393
|
+
expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
|
394
|
+
expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
|
395
|
+
expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
|
396
|
+
expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
|
399
397
|
end
|
400
398
|
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
}
|
412
|
-
}
|
413
|
-
]
|
414
|
-
|
415
|
-
pipeline = new_pipeline_from_string(config_metadata_with_string)
|
416
|
-
queue = Queue.new
|
417
|
-
pipeline.instance_eval do
|
418
|
-
@output_func = lambda { |event| queue << event }
|
399
|
+
context 'if the `docinfo_target` exist but is not of type hash' do
|
400
|
+
let (:config) { {
|
401
|
+
"hosts" => ["localhost"],
|
402
|
+
"query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
|
403
|
+
"docinfo" => true,
|
404
|
+
"docinfo_target" => 'metadata_with_string'
|
405
|
+
} }
|
406
|
+
it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
|
407
|
+
plugin.register
|
408
|
+
expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
|
419
409
|
end
|
420
|
-
|
421
|
-
expect { pipeline.run }.to raise_error(Exception, /incompatible event/)
|
422
410
|
end
|
423
411
|
|
424
412
|
it "should move the document info to the @metadata field" do
|
@@ -516,6 +504,80 @@ describe LogStash::Inputs::Elasticsearch do
|
|
516
504
|
end
|
517
505
|
end
|
518
506
|
|
507
|
+
describe "client" do
|
508
|
+
let(:config) do
|
509
|
+
{
|
510
|
+
|
511
|
+
}
|
512
|
+
end
|
513
|
+
let(:plugin) { described_class.new(config) }
|
514
|
+
let(:event) { LogStash::Event.new({}) }
|
515
|
+
|
516
|
+
describe "cloud.id" do
|
517
|
+
let(:valid_cloud_id) do
|
518
|
+
'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
|
519
|
+
end
|
520
|
+
|
521
|
+
let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
|
522
|
+
|
523
|
+
it "should set host(s)" do
|
524
|
+
plugin.register
|
525
|
+
client = plugin.send(:client)
|
526
|
+
expect( client.transport.hosts ).to eql [{
|
527
|
+
:scheme => "https",
|
528
|
+
:host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
|
529
|
+
:port => 9243,
|
530
|
+
:path => "",
|
531
|
+
:protocol => "https"
|
532
|
+
}]
|
533
|
+
end
|
534
|
+
|
535
|
+
context 'invalid' do
|
536
|
+
let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
|
537
|
+
|
538
|
+
it "should fail" do
|
539
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
|
540
|
+
end
|
541
|
+
end
|
542
|
+
|
543
|
+
context 'hosts also set' do
|
544
|
+
let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
|
545
|
+
|
546
|
+
it "should fail" do
|
547
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
|
548
|
+
end
|
549
|
+
end
|
550
|
+
end if LOGSTASH_VERSION > '6.0'
|
551
|
+
|
552
|
+
describe "cloud.auth" do
|
553
|
+
let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
|
554
|
+
|
555
|
+
it "should set authorization" do
|
556
|
+
plugin.register
|
557
|
+
client = plugin.send(:client)
|
558
|
+
auth_header = client.transport.options[:transport_options][:headers][:Authorization]
|
559
|
+
|
560
|
+
expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
|
561
|
+
end
|
562
|
+
|
563
|
+
context 'invalid' do
|
564
|
+
let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
|
565
|
+
|
566
|
+
it "should fail" do
|
567
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
|
568
|
+
end
|
569
|
+
end
|
570
|
+
|
571
|
+
context 'user also set' do
|
572
|
+
let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
|
573
|
+
|
574
|
+
it "should fail" do
|
575
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
|
576
|
+
end
|
577
|
+
end
|
578
|
+
end if LOGSTASH_VERSION > '6.0'
|
579
|
+
end
|
580
|
+
|
519
581
|
context "when scheduling" do
|
520
582
|
let(:config) do
|
521
583
|
{
|
@@ -525,39 +587,11 @@ describe LogStash::Inputs::Elasticsearch do
|
|
525
587
|
}
|
526
588
|
end
|
527
589
|
|
528
|
-
response = {
|
529
|
-
"_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
|
530
|
-
"took" => 27,
|
531
|
-
"timed_out" => false,
|
532
|
-
"_shards" => {
|
533
|
-
"total" => 169,
|
534
|
-
"successful" => 169,
|
535
|
-
"failed" => 0
|
536
|
-
},
|
537
|
-
"hits" => {
|
538
|
-
"total" => 1,
|
539
|
-
"max_score" => 1.0,
|
540
|
-
"hits" => [ {
|
541
|
-
"_index" => "logstash-2014.10.12",
|
542
|
-
"_type" => "logs",
|
543
|
-
"_id" => "C5b2xLQwTZa76jBmHIbwHQ",
|
544
|
-
"_score" => 1.0,
|
545
|
-
"_source" => { "message" => ["ohayo"] }
|
546
|
-
} ]
|
547
|
-
}
|
548
|
-
}
|
549
|
-
|
550
|
-
scroll_reponse = {
|
551
|
-
"_scroll_id" => "r453Wc1jh0caLJhSDg",
|
552
|
-
"hits" => { "hits" => [] }
|
553
|
-
}
|
554
|
-
|
555
590
|
before do
|
556
591
|
plugin.register
|
557
592
|
end
|
558
593
|
|
559
594
|
it "should properly schedule" do
|
560
|
-
|
561
595
|
Timecop.travel(Time.new(2000))
|
562
596
|
Timecop.scale(60)
|
563
597
|
runner = Thread.new do
|
@@ -576,5 +610,4 @@ describe LogStash::Inputs::Elasticsearch do
|
|
576
610
|
end
|
577
611
|
|
578
612
|
end
|
579
|
-
|
580
613
|
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/plugin"
|
4
|
+
require "logstash/inputs/elasticsearch"
|
5
|
+
require_relative "../../../spec/es_helper"
|
6
|
+
|
7
|
+
describe LogStash::Inputs::Elasticsearch do
|
8
|
+
|
9
|
+
let(:config) { { 'hosts' => [ESHelper.get_host_port],
|
10
|
+
'index' => 'logs',
|
11
|
+
'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
|
12
|
+
let(:plugin) { described_class.new(config) }
|
13
|
+
let(:event) { LogStash::Event.new({}) }
|
14
|
+
let(:client_options) {{}}
|
15
|
+
|
16
|
+
before(:each) do
|
17
|
+
@es = ESHelper.get_client(client_options)
|
18
|
+
# Delete all templates first.
|
19
|
+
# Clean ES of data before we start.
|
20
|
+
@es.indices.delete_template(:name => "*")
|
21
|
+
# This can fail if there are no indexes, ignore failure.
|
22
|
+
@es.indices.delete(:index => "*") rescue nil
|
23
|
+
10.times do
|
24
|
+
ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
|
25
|
+
end
|
26
|
+
@es.indices.refresh
|
27
|
+
plugin.register
|
28
|
+
end
|
29
|
+
|
30
|
+
after(:each) do
|
31
|
+
@es.indices.delete_template(:name => "*")
|
32
|
+
@es.indices.delete(:index => "*") rescue nil
|
33
|
+
end
|
34
|
+
|
35
|
+
shared_examples 'an elasticsearch index plugin' do
|
36
|
+
it 'should retrieve json event from elasticsearch' do
|
37
|
+
queue = []
|
38
|
+
plugin.run(queue)
|
39
|
+
event = queue.pop
|
40
|
+
expect(event).to be_a(LogStash::Event)
|
41
|
+
expect(event.get("response")).to eql(404)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
describe 'against an unsecured elasticsearch', :integration => true do
|
46
|
+
it_behaves_like 'an elasticsearch index plugin'
|
47
|
+
end
|
48
|
+
|
49
|
+
describe 'against a secured elasticsearch', :secure_integration => true do
|
50
|
+
let(:user) { 'simpleuser' }
|
51
|
+
let(:password) { 'abc123' }
|
52
|
+
let(:ca_file) { "spec/fixtures/test_certs/test.crt" }
|
53
|
+
let(:client_options) {{:ca_file => ca_file, :user => user, :password => password}}
|
54
|
+
let(:config) { super.merge({
|
55
|
+
'user' => user,
|
56
|
+
'password' => password,
|
57
|
+
'ssl' => true,
|
58
|
+
'ca_file' => ca_file })
|
59
|
+
}
|
60
|
+
it_behaves_like 'an elasticsearch index plugin'
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -36,9 +36,6 @@ dependencies:
|
|
36
36
|
- - ">="
|
37
37
|
- !ruby/object:Gem::Version
|
38
38
|
version: 5.0.3
|
39
|
-
- - "<"
|
40
|
-
- !ruby/object:Gem::Version
|
41
|
-
version: 6.0.0
|
42
39
|
name: elasticsearch
|
43
40
|
prerelease: false
|
44
41
|
type: :runtime
|
@@ -47,9 +44,6 @@ dependencies:
|
|
47
44
|
- - ">="
|
48
45
|
- !ruby/object:Gem::Version
|
49
46
|
version: 5.0.3
|
50
|
-
- - "<"
|
51
|
-
- !ruby/object:Gem::Version
|
52
|
-
version: 6.0.0
|
53
47
|
- !ruby/object:Gem::Dependency
|
54
48
|
requirement: !ruby/object:Gem::Requirement
|
55
49
|
requirements:
|
@@ -134,6 +128,34 @@ dependencies:
|
|
134
128
|
- - ">="
|
135
129
|
- !ruby/object:Gem::Version
|
136
130
|
version: '0'
|
131
|
+
- !ruby/object:Gem::Dependency
|
132
|
+
requirement: !ruby/object:Gem::Requirement
|
133
|
+
requirements:
|
134
|
+
- - "~>"
|
135
|
+
- !ruby/object:Gem::Version
|
136
|
+
version: '0.6'
|
137
|
+
name: manticore
|
138
|
+
prerelease: false
|
139
|
+
type: :runtime
|
140
|
+
version_requirements: !ruby/object:Gem::Requirement
|
141
|
+
requirements:
|
142
|
+
- - "~>"
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0.6'
|
145
|
+
- !ruby/object:Gem::Dependency
|
146
|
+
requirement: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - "~>"
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: 0.15.4
|
151
|
+
name: faraday
|
152
|
+
prerelease: false
|
153
|
+
type: :runtime
|
154
|
+
version_requirements: !ruby/object:Gem::Requirement
|
155
|
+
requirements:
|
156
|
+
- - "~>"
|
157
|
+
- !ruby/object:Gem::Version
|
158
|
+
version: 0.15.4
|
137
159
|
- !ruby/object:Gem::Dependency
|
138
160
|
requirement: !ruby/object:Gem::Requirement
|
139
161
|
requirements:
|
@@ -179,7 +201,13 @@ files:
|
|
179
201
|
- docs/index.asciidoc
|
180
202
|
- lib/logstash/inputs/elasticsearch.rb
|
181
203
|
- logstash-input-elasticsearch.gemspec
|
204
|
+
- spec/es_helper.rb
|
205
|
+
- spec/fixtures/test_certs/ca/ca.crt
|
206
|
+
- spec/fixtures/test_certs/ca/ca.key
|
207
|
+
- spec/fixtures/test_certs/test.crt
|
208
|
+
- spec/fixtures/test_certs/test.key
|
182
209
|
- spec/inputs/elasticsearch_spec.rb
|
210
|
+
- spec/inputs/integration/elasticsearch_spec.rb
|
183
211
|
homepage: http://www.elastic.co/guide/en/logstash/current/index.html
|
184
212
|
licenses:
|
185
213
|
- Apache License (2.0)
|
@@ -207,4 +235,10 @@ signing_key:
|
|
207
235
|
specification_version: 4
|
208
236
|
summary: Reads query results from an Elasticsearch cluster
|
209
237
|
test_files:
|
238
|
+
- spec/es_helper.rb
|
239
|
+
- spec/fixtures/test_certs/ca/ca.crt
|
240
|
+
- spec/fixtures/test_certs/ca/ca.key
|
241
|
+
- spec/fixtures/test_certs/test.crt
|
242
|
+
- spec/fixtures/test_certs/test.key
|
210
243
|
- spec/inputs/elasticsearch_spec.rb
|
244
|
+
- spec/inputs/integration/elasticsearch_spec.rb
|