logstash-input-elasticsearch 4.4.0 → 4.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/docs/index.asciidoc +23 -2
- data/lib/logstash/inputs/elasticsearch.rb +79 -3
- data/logstash-input-elasticsearch.gemspec +1 -1
- data/spec/inputs/elasticsearch_spec.rb +74 -29
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a5e1d79e9b4b6fdd1deaf7575f6fe3c5c91c578a327478d3dec2172ca3d12cc0
|
4
|
+
data.tar.gz: 046771cceceb57a4a9f0ad395f55fa273dc80b224fa50eab410c3a3b0212ff2b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 24e767e7f20fe2320ad6e103762e0a5158d23d3c63ed781675d1c9fd648bea8fcbc654722bd2c792cedccdc5f534617ac78d552938e60a8b7c957e5b078e1d70
|
7
|
+
data.tar.gz: 9133d597e944199974171430a9c6bbe1acc7c3709d5d6371176e26152d53b0256dc62cac045e614f8123e10b904aa5af9c31a1d6c58be47b83823c0762d95fda
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
+
## 4.5.0
|
2
|
+
- Feat: Added support for cloud_id / cloud_auth configuration [#112](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/112)
|
3
|
+
|
1
4
|
## 4.4.0
|
2
|
-
- Changed Elasticsearch Client transport to use Manticore[#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
|
5
|
+
- Changed Elasticsearch Client transport to use Manticore [#111](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/111)
|
3
6
|
|
4
7
|
## 4.3.3
|
5
8
|
- Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
|
data/docs/index.asciidoc
CHANGED
@@ -87,6 +87,8 @@ This plugin supports the following configuration options plus the <<plugins-{typ
|
|
87
87
|
|=======================================================================
|
88
88
|
|Setting |Input type|Required
|
89
89
|
| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|No
|
90
|
+
| <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
|
91
|
+
| <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
|
90
92
|
| <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
|
91
93
|
| <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
|
92
94
|
| <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
|
@@ -113,8 +115,27 @@ input plugins.
|
|
113
115
|
* Value type is <<path,path>>
|
114
116
|
* There is no default value for this setting.
|
115
117
|
|
116
|
-
SSL Certificate Authority file in PEM encoded format, must also
|
117
|
-
|
118
|
+
SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
|
119
|
+
|
120
|
+
[id="plugins-{type}s-{plugin}-cloud_auth"]
|
121
|
+
===== `cloud_auth`
|
122
|
+
|
123
|
+
* Value type is <<password,password>>
|
124
|
+
* There is no default value for this setting.
|
125
|
+
|
126
|
+
Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` pair.
|
127
|
+
|
128
|
+
For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
|
129
|
+
|
130
|
+
[id="plugins-{type}s-{plugin}-cloud_id"]
|
131
|
+
===== `cloud_id`
|
132
|
+
|
133
|
+
* Value type is <<string,string>>
|
134
|
+
* There is no default value for this setting.
|
135
|
+
|
136
|
+
Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
|
137
|
+
|
138
|
+
For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
|
118
139
|
|
119
140
|
[id="plugins-{type}s-{plugin}-docinfo"]
|
120
141
|
===== `docinfo`
|
@@ -2,6 +2,7 @@
|
|
2
2
|
require "logstash/inputs/base"
|
3
3
|
require "logstash/namespace"
|
4
4
|
require "logstash/json"
|
5
|
+
require "logstash/util/safe_uri"
|
5
6
|
require "base64"
|
6
7
|
|
7
8
|
|
@@ -69,6 +70,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
69
70
|
# Port defaults to 9200
|
70
71
|
config :hosts, :validate => :array
|
71
72
|
|
73
|
+
# Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
|
74
|
+
#
|
75
|
+
# For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation]
|
76
|
+
config :cloud_id, :validate => :string
|
77
|
+
|
72
78
|
# The index or alias to search.
|
73
79
|
config :index, :validate => :string, :default => "logstash-*"
|
74
80
|
|
@@ -134,6 +140,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
134
140
|
# Basic Auth - password
|
135
141
|
config :password, :validate => :password
|
136
142
|
|
143
|
+
# Cloud authentication string ("<username>:<password>" format) is an alternative for the `user`/`password` configuration.
|
144
|
+
#
|
145
|
+
# For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation]
|
146
|
+
config :cloud_auth, :validate => :password
|
147
|
+
|
137
148
|
# SSL
|
138
149
|
config :ssl, :validate => :boolean, :default => false
|
139
150
|
|
@@ -165,12 +176,17 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
165
176
|
|
166
177
|
transport_options = {}
|
167
178
|
|
179
|
+
fill_user_password_from_cloud_auth
|
180
|
+
|
168
181
|
if @user && @password
|
169
182
|
token = Base64.strict_encode64("#{@user}:#{@password.value}")
|
170
183
|
transport_options[:headers] = { :Authorization => "Basic #{token}" }
|
171
184
|
end
|
172
185
|
|
173
|
-
|
186
|
+
fill_hosts_from_cloud_id
|
187
|
+
@hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
|
188
|
+
|
189
|
+
hosts = if @ssl
|
174
190
|
@hosts.map do |h|
|
175
191
|
host, port = h.split(":")
|
176
192
|
{ :host => host, :scheme => 'https', :port => port }
|
@@ -275,10 +291,70 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
|
|
275
291
|
end
|
276
292
|
|
277
293
|
def scroll_request scroll_id
|
278
|
-
|
294
|
+
client.scroll(:body => { :scroll_id => scroll_id }, :scroll => @scroll)
|
279
295
|
end
|
280
296
|
|
281
297
|
def search_request(options)
|
282
|
-
|
298
|
+
client.search(options)
|
299
|
+
end
|
300
|
+
|
301
|
+
attr_reader :client
|
302
|
+
|
303
|
+
def hosts_default?(hosts)
|
304
|
+
hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? )
|
305
|
+
end
|
306
|
+
|
307
|
+
def fill_hosts_from_cloud_id
|
308
|
+
return unless @cloud_id
|
309
|
+
|
310
|
+
if @hosts && !hosts_default?(@hosts)
|
311
|
+
raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.'
|
312
|
+
end
|
313
|
+
@hosts = parse_host_uri_from_cloud_id(@cloud_id)
|
314
|
+
end
|
315
|
+
|
316
|
+
def fill_user_password_from_cloud_auth
|
317
|
+
return unless @cloud_auth
|
318
|
+
|
319
|
+
if @user || @password
|
320
|
+
raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.'
|
321
|
+
end
|
322
|
+
@user, @password = parse_user_password_from_cloud_auth(@cloud_auth)
|
323
|
+
params['user'], params['password'] = @user, @password
|
324
|
+
end
|
325
|
+
|
326
|
+
def parse_host_uri_from_cloud_id(cloud_id)
|
327
|
+
begin # might not be available on older LS
|
328
|
+
require 'logstash/util/cloud_setting_id'
|
329
|
+
rescue LoadError
|
330
|
+
raise LogStash::ConfigurationError, 'The cloud_id setting is not supported by your version of Logstash, ' +
|
331
|
+
'please upgrade your installation (or set hosts instead).'
|
332
|
+
end
|
333
|
+
|
334
|
+
begin
|
335
|
+
cloud_id = LogStash::Util::CloudSettingId.new(cloud_id) # already does append ':{port}' to host
|
336
|
+
rescue ArgumentError => e
|
337
|
+
raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Id/i, 'cloud_id')
|
338
|
+
end
|
339
|
+
cloud_uri = "#{cloud_id.elasticsearch_scheme}://#{cloud_id.elasticsearch_host}"
|
340
|
+
LogStash::Util::SafeURI.new(cloud_uri)
|
283
341
|
end
|
342
|
+
|
343
|
+
def parse_user_password_from_cloud_auth(cloud_auth)
|
344
|
+
begin # might not be available on older LS
|
345
|
+
require 'logstash/util/cloud_setting_auth'
|
346
|
+
rescue LoadError
|
347
|
+
raise LogStash::ConfigurationError, 'The cloud_auth setting is not supported by your version of Logstash, ' +
|
348
|
+
'please upgrade your installation (or set user/password instead).'
|
349
|
+
end
|
350
|
+
|
351
|
+
cloud_auth = cloud_auth.value if cloud_auth.is_a?(LogStash::Util::Password)
|
352
|
+
begin
|
353
|
+
cloud_auth = LogStash::Util::CloudSettingAuth.new(cloud_auth)
|
354
|
+
rescue ArgumentError => e
|
355
|
+
raise LogStash::ConfigurationError, e.message.to_s.sub(/Cloud Auth/i, 'cloud_auth')
|
356
|
+
end
|
357
|
+
[ cloud_auth.username, cloud_auth.password ]
|
358
|
+
end
|
359
|
+
|
284
360
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-input-elasticsearch'
|
4
|
-
s.version = '4.
|
4
|
+
s.version = '4.5.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Reads query results from an Elasticsearch cluster"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -504,6 +504,80 @@ describe LogStash::Inputs::Elasticsearch do
|
|
504
504
|
end
|
505
505
|
end
|
506
506
|
|
507
|
+
describe "client" do
|
508
|
+
let(:config) do
|
509
|
+
{
|
510
|
+
|
511
|
+
}
|
512
|
+
end
|
513
|
+
let(:plugin) { described_class.new(config) }
|
514
|
+
let(:event) { LogStash::Event.new({}) }
|
515
|
+
|
516
|
+
describe "cloud.id" do
|
517
|
+
let(:valid_cloud_id) do
|
518
|
+
'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
|
519
|
+
end
|
520
|
+
|
521
|
+
let(:config) { super.merge({ 'cloud_id' => valid_cloud_id }) }
|
522
|
+
|
523
|
+
it "should set host(s)" do
|
524
|
+
plugin.register
|
525
|
+
client = plugin.send(:client)
|
526
|
+
expect( client.transport.hosts ).to eql [{
|
527
|
+
:scheme => "https",
|
528
|
+
:host => "ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io",
|
529
|
+
:port => 9243,
|
530
|
+
:path => "",
|
531
|
+
:protocol => "https"
|
532
|
+
}]
|
533
|
+
end
|
534
|
+
|
535
|
+
context 'invalid' do
|
536
|
+
let(:config) { super.merge({ 'cloud_id' => 'invalid:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlv' }) }
|
537
|
+
|
538
|
+
it "should fail" do
|
539
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id.*? is invalid/
|
540
|
+
end
|
541
|
+
end
|
542
|
+
|
543
|
+
context 'hosts also set' do
|
544
|
+
let(:config) { super.merge({ 'cloud_id' => valid_cloud_id, 'hosts' => [ 'localhost:9200' ] }) }
|
545
|
+
|
546
|
+
it "should fail" do
|
547
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_id and hosts/
|
548
|
+
end
|
549
|
+
end
|
550
|
+
end if LOGSTASH_VERSION > '6.0'
|
551
|
+
|
552
|
+
describe "cloud.auth" do
|
553
|
+
let(:config) { super.merge({ 'cloud_auth' => LogStash::Util::Password.new('elastic:my-passwd-00') }) }
|
554
|
+
|
555
|
+
it "should set authorization" do
|
556
|
+
plugin.register
|
557
|
+
client = plugin.send(:client)
|
558
|
+
auth_header = client.transport.options[:transport_options][:headers][:Authorization]
|
559
|
+
|
560
|
+
expect( auth_header ).to eql "Basic #{Base64.encode64('elastic:my-passwd-00').rstrip}"
|
561
|
+
end
|
562
|
+
|
563
|
+
context 'invalid' do
|
564
|
+
let(:config) { super.merge({ 'cloud_auth' => 'invalid-format' }) }
|
565
|
+
|
566
|
+
it "should fail" do
|
567
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth.*? format/
|
568
|
+
end
|
569
|
+
end
|
570
|
+
|
571
|
+
context 'user also set' do
|
572
|
+
let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) }
|
573
|
+
|
574
|
+
it "should fail" do
|
575
|
+
expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/
|
576
|
+
end
|
577
|
+
end
|
578
|
+
end if LOGSTASH_VERSION > '6.0'
|
579
|
+
end
|
580
|
+
|
507
581
|
context "when scheduling" do
|
508
582
|
let(:config) do
|
509
583
|
{
|
@@ -513,39 +587,11 @@ describe LogStash::Inputs::Elasticsearch do
|
|
513
587
|
}
|
514
588
|
end
|
515
589
|
|
516
|
-
response = {
|
517
|
-
"_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
|
518
|
-
"took" => 27,
|
519
|
-
"timed_out" => false,
|
520
|
-
"_shards" => {
|
521
|
-
"total" => 169,
|
522
|
-
"successful" => 169,
|
523
|
-
"failed" => 0
|
524
|
-
},
|
525
|
-
"hits" => {
|
526
|
-
"total" => 1,
|
527
|
-
"max_score" => 1.0,
|
528
|
-
"hits" => [ {
|
529
|
-
"_index" => "logstash-2014.10.12",
|
530
|
-
"_type" => "logs",
|
531
|
-
"_id" => "C5b2xLQwTZa76jBmHIbwHQ",
|
532
|
-
"_score" => 1.0,
|
533
|
-
"_source" => { "message" => ["ohayo"] }
|
534
|
-
} ]
|
535
|
-
}
|
536
|
-
}
|
537
|
-
|
538
|
-
scroll_reponse = {
|
539
|
-
"_scroll_id" => "r453Wc1jh0caLJhSDg",
|
540
|
-
"hits" => { "hits" => [] }
|
541
|
-
}
|
542
|
-
|
543
590
|
before do
|
544
591
|
plugin.register
|
545
592
|
end
|
546
593
|
|
547
594
|
it "should properly schedule" do
|
548
|
-
|
549
595
|
Timecop.travel(Time.new(2000))
|
550
596
|
Timecop.scale(60)
|
551
597
|
runner = Thread.new do
|
@@ -564,5 +610,4 @@ describe LogStash::Inputs::Elasticsearch do
|
|
564
610
|
end
|
565
611
|
|
566
612
|
end
|
567
|
-
|
568
613
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|