logstash-output-elasticsearch 11.0.3-java → 11.2.0-java

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 143b78ff484990a35c4d9a1b54ccb040bdba7e6aa2fd69928b0553544883fabd
4
- data.tar.gz: a14e65b2b499d65d9a8f3389d0c0168f0447e1358338d867dd5b05a78ca28c13
3
+ metadata.gz: 0bd2f7b95f4a121df8712ce601a2c56c228de6e113f90c96058c58f966af31db
4
+ data.tar.gz: bac417c7d999e1676ed451cf5216b0b6085203b533fd7d5039abf3efbc070578
5
5
  SHA512:
6
- metadata.gz: fd6495287f9d5dff77d6006c5495013335eb869b7a9678973567358b9ce6ed772822cda5b38bfbeff1831e72406a4e550125cfafd9cbaeab20b588ccf584a7b6
7
- data.tar.gz: d6eda43b3338ad401b889ff20f7bd246b3a44f921b0f006989572cb95145a518cb7103a1c23fd930018c2552316af33d356d4fecc8210ca95f0a6259a2b1037f
6
+ metadata.gz: '09274b381e8026eb8527b5926318c603ffc776e645538671263f61fec3d00731e7695a13e2f72cdccd4037f1d7298975a6424f03115a38060f4e2e34f7498049'
7
+ data.tar.gz: 47fddb97c3634cd68588bebaed1d0fac38d4eb33c3cc69083e59a9c13c1f74a87f7835e6fc7982762a2db19009150900ca7ac2bee213d973e56dc0a8fbcc1158
data/CHANGELOG.md CHANGED
@@ -1,3 +1,15 @@
1
+ ## 11.2.0
2
+ - Added preflight checks on Elasticsearch [#1026](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1026)
3
+
4
+ ## 11.1.0
5
+ - Feat: add `user-agent` header passed to the Elasticsearch HTTP connection [#1038](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1038)
6
+
7
+ ## 11.0.5
8
+ - Fixed running post-register action when Elasticsearch status change from unhealthy to healthy [#1035](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1035)
9
+
10
+ ## 11.0.4
11
+ - [DOC] Clarify that `http_compression` applies to _requests_, and remove noise about _response_ decompression [#1000](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1000)
12
+
1
13
  ## 11.0.3
2
14
  - Fixed SSL handshake hang indefinitely with proxy setup [#1032](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1032)
3
15
 
data/docs/index.asciidoc CHANGED
@@ -278,16 +278,11 @@ not reevaluate its DNS value while the keepalive is in effect.
278
278
 
279
279
  ==== HTTP Compression
280
280
 
281
- This plugin supports request and response compression. Response compression is
282
- enabled by default for HTTP and for Elasticsearch versions 5.0 and later.
281
+ This plugin supports request compression, and handles compressed responses
282
+ from Elasticsearch.
283
283
 
284
- You don't have to set any configs in Elasticsearch for it to send back a
285
- compressed response. For versions before 5.0, or if HTTPS is enabled,
286
- `http.compression` must be set to `true` {ref}/modules-http.html#modules-http[in
287
- Elasticsearch] to take advantage of response compression when using this plugin.
288
-
289
- For requests compression, regardless of the Elasticsearch version, enable the
290
- `http_compression` setting in the Logstash config file.
284
+ To enable request compression, use the <<plugins-{type}s-{plugin}-http_compression>>
285
+ setting on this plugin.
291
286
 
292
287
  ==== Authentication
293
288
 
@@ -640,8 +635,13 @@ Any special characters present in the URLs here MUST be URL escaped! This means
640
635
  * Value type is <<boolean,boolean>>
641
636
  * Default value is `false`
642
637
 
643
- Enable gzip compression on requests. Note that response compression is on by
644
- default for Elasticsearch v5.0 and beyond
638
+ Enable gzip compression on requests.
639
+
640
+ This setting allows you to reduce this plugin's outbound network traffic by
641
+ compressing each bulk _request_ to {es}.
642
+
643
+ NOTE: This output plugin reads compressed _responses_ from {es} regardless
644
+ of the value of this setting.
645
645
 
646
646
  [id="plugins-{type}s-{plugin}-ilm_enabled"]
647
647
  ===== `ilm_enabled`
@@ -37,6 +37,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
37
37
  ROOT_URI_PATH = '/'.freeze
38
38
  LICENSE_PATH = '/_license'.freeze
39
39
 
40
+ VERSION_6_TO_7 = Gem::Requirement.new([">= 6.0.0", "< 7.0.0"])
41
+ VERSION_7_TO_7_14 = Gem::Requirement.new([">= 7.0.0", "< 7.14.0"])
42
+
40
43
  DEFAULT_OPTIONS = {
41
44
  :healthcheck_path => ROOT_URI_PATH,
42
45
  :sniffing_path => "/_nodes/http",
@@ -211,7 +214,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
211
214
  def start_resurrectionist
212
215
  @resurrectionist = Thread.new do
213
216
  until_stopped("resurrection", @resurrect_delay) do
214
- healthcheck!
217
+ healthcheck!(false)
215
218
  end
216
219
  end
217
220
  end
@@ -232,11 +235,18 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
232
235
  perform_request_to_url(url, :head, @healthcheck_path)
233
236
  end
234
237
 
235
- def healthcheck!
238
+ def healthcheck!(register_phase = true)
236
239
  # Try to keep locking granularity low such that we don't affect IO...
237
240
  @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
238
241
  begin
239
242
  health_check_request(url)
243
+
244
+ # when called from resurrectionist skip the product check done during register phase
245
+ if register_phase
246
+ if !elasticsearch?(url)
247
+ raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
248
+ end
249
+ end
240
250
  # If no exception was raised it must have succeeded!
241
251
  logger.warn("Restored connection to ES instance", url: url.sanitized.to_s)
242
252
  # We reconnected to this node, check its ES version
@@ -254,6 +264,42 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
254
264
  end
255
265
  end
256
266
 
267
+ def elasticsearch?(url)
268
+ begin
269
+ response = perform_request_to_url(url, :get, ROOT_URI_PATH)
270
+ rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
271
+ return false if response.code == 401 || response.code == 403
272
+ raise e
273
+ end
274
+
275
+ version_info = LogStash::Json.load(response.body)
276
+ return false if version_info['version'].nil?
277
+
278
+ version = Gem::Version.new(version_info["version"]['number'])
279
+ return false if version < Gem::Version.new('6.0.0')
280
+
281
+ if VERSION_6_TO_7.satisfied_by?(version)
282
+ return valid_tagline?(version_info)
283
+ elsif VERSION_7_TO_7_14.satisfied_by?(version)
284
+ build_flavor = version_info["version"]['build_flavor']
285
+ return false if build_flavor.nil? || build_flavor != 'default' || !valid_tagline?(version_info)
286
+ else
287
+ # case >= 7.14
288
+ lower_headers = response.headers.transform_keys {|key| key.to_s.downcase }
289
+ product_header = lower_headers['x-elastic-product']
290
+ return false if product_header != 'Elasticsearch'
291
+ end
292
+ return true
293
+ rescue => e
294
+ logger.error("Unable to retrieve Elasticsearch version", url: url.sanitized.to_s, exception: e.class, message: e.message)
295
+ false
296
+ end
297
+
298
+ def valid_tagline?(version_info)
299
+ tagline = version_info['tagline']
300
+ tagline == "You Know, for Search"
301
+ end
302
+
257
303
  def stop_resurrectionist
258
304
  @resurrectionist.join if @resurrectionist
259
305
  end
@@ -4,6 +4,7 @@ require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
4
4
  require 'cgi'
5
5
  require 'zlib'
6
6
  require 'stringio'
7
+ require 'java'
7
8
 
8
9
  module LogStash; module Outputs; class ElasticSearch;
9
10
  # This is a constant instead of a config option because
@@ -92,6 +93,10 @@ module LogStash; module Outputs; class ElasticSearch;
92
93
  @pool.maximum_seen_major_version
93
94
  end
94
95
 
96
+ def alive_urls_count
97
+ @pool.alive_urls_count
98
+ end
99
+
95
100
  def bulk(actions)
96
101
  @action_count ||= 0
97
102
  @action_count += actions.size
@@ -297,6 +302,8 @@ module LogStash; module Outputs; class ElasticSearch;
297
302
  :request_timeout => timeout,
298
303
  }
299
304
 
305
+ adapter_options[:user_agent] = prepare_user_agent
306
+
300
307
  adapter_options[:proxy] = client_settings[:proxy] if client_settings[:proxy]
301
308
 
302
309
  adapter_options[:check_connection_timeout] = client_settings[:check_connection_timeout] if client_settings[:check_connection_timeout]
@@ -318,6 +325,18 @@ module LogStash; module Outputs; class ElasticSearch;
318
325
  adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
319
326
  adapter = adapter_class.new(@logger, adapter_options)
320
327
  end
328
+
329
+ def prepare_user_agent
330
+ os_name = java.lang.System.getProperty('os.name')
331
+ os_version = java.lang.System.getProperty('os.version')
332
+ os_arch = java.lang.System.getProperty('os.arch')
333
+ jvm_vendor = java.lang.System.getProperty('java.vendor')
334
+ jvm_version = java.lang.System.getProperty('java.version')
335
+
336
+ plugin_version = Gem.loaded_specs['logstash-output-elasticsearch'].version
337
+ # example: Logstash/7.14.1 (OS=Linux-5.4.0-84-generic-amd64; JVM=AdoptOpenJDK-11.0.11) logstash-output-elasticsearch/11.0.1
338
+ "Logstash/#{LOGSTASH_VERSION} (OS=#{os_name}-#{os_version}-#{os_arch}; JVM=#{jvm_vendor}-#{jvm_version}) logstash-output-elasticsearch/#{plugin_version}"
339
+ end
321
340
 
322
341
  def build_pool(options)
323
342
  adapter = build_adapter(options)
@@ -128,8 +128,12 @@ module LogStash; module PluginMixins; module ElasticSearch
128
128
  client.maximum_seen_major_version
129
129
  end
130
130
 
131
+ def alive_urls_count
132
+ client.alive_urls_count
133
+ end
134
+
131
135
  def successful_connection?
132
- !!maximum_seen_major_version
136
+ !!maximum_seen_major_version && alive_urls_count > 0
133
137
  end
134
138
 
135
139
  # launch a thread that waits for an initial successful connection to the ES cluster to call the given block
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '11.0.3'
3
+ s.version = '11.2.0'
4
4
 
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Stores logs in Elasticsearch"
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
30
30
  s.add_development_dependency 'logstash-devutils'
31
31
  s.add_development_dependency 'flores'
32
32
  s.add_development_dependency 'cabin', ['~> 0.6']
33
+ s.add_development_dependency 'webrick'
34
+ s.add_development_dependency 'webmock'
33
35
  # Still used in some specs, we should remove this ASAP
34
36
  s.add_development_dependency 'elasticsearch'
35
37
  end
@@ -80,7 +80,7 @@ module ESHelper
80
80
  end
81
81
 
82
82
  def self.es_version_satisfies?(*requirement)
83
- es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
83
+ es_version = nilify(RSpec.configuration.filter[:es_version]) || nilify(ENV['ES_VERSION']) || nilify(ENV['ELASTIC_STACK_VERSION'])
84
84
  if es_version.nil?
85
85
  puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
86
86
  return false
@@ -89,6 +89,15 @@ module ESHelper
89
89
  Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
90
90
  end
91
91
 
92
+ private
93
+ def self.nilify(str)
94
+ if str.nil?
95
+ return str
96
+ end
97
+ str.empty? ? nil : str
98
+ end
99
+
100
+ public
92
101
  def clean(client)
93
102
  client.indices.delete_template(:name => "*")
94
103
  client.indices.delete_index_template(:name => "logstash*") rescue nil
@@ -55,4 +55,18 @@ describe "elasticsearch is down on startup", :integration => true do
55
55
  expect(r).to have_hits(2)
56
56
  end
57
57
 
58
+ it 'should get cluster_uuid when Elasticsearch recovers from license check failure' do
59
+ allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).with(instance_of(LogStash::Util::SafeURI)).and_raise(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError.new(StandardError.new, "big fail"))
60
+ subject.register
61
+ Thread.new do
62
+ sleep 4
63
+ allow_any_instance_of(LogStash::Outputs::ElasticSearch::HttpClient::Pool).to receive(:get_license).with(instance_of(LogStash::Util::SafeURI)).and_call_original
64
+ end
65
+ subject.multi_receive([event1, event2])
66
+ @es.indices.refresh
67
+ r = @es.search(index: 'logstash-*')
68
+ expect(r).to have_hits(2)
69
+ expect(subject.plugin_metadata.get(:cluster_uuid)).not_to be_empty
70
+ expect(subject.plugin_metadata.get(:cluster_uuid)).not_to eq("_na_")
71
+ end if ESHelper.es_version_satisfies?(">=7")
58
72
  end
@@ -50,15 +50,18 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
50
50
 
51
51
  describe "healthcheck url handling" do
52
52
  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
53
+ before(:example) do
54
+ expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
55
+ expect(url.path).to be_empty
56
+ end
57
+ end
53
58
 
54
59
  context "and not setting healthcheck_path" do
55
60
  it "performs the healthcheck to the root" do
56
- expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
57
- expect(method).to eq(:head)
61
+ expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
58
62
  expect(url.path).to be_empty
59
- expect(req_path).to eq("/")
60
63
  end
61
- subject.healthcheck!
64
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
62
65
  end
63
66
  end
64
67
 
@@ -66,12 +69,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
66
69
  let(:healthcheck_path) { "/my/health" }
67
70
  let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
68
71
  it "performs the healthcheck to the healthcheck_path" do
69
- expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
70
- expect(method).to eq(:head)
72
+ expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
71
73
  expect(url.path).to be_empty
72
- expect(req_path).to eq(healthcheck_path)
73
74
  end
74
- subject.healthcheck!
75
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
75
76
  end
76
77
  end
77
78
  end
@@ -164,6 +165,20 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
164
165
  end
165
166
  end
166
167
 
168
+ class MockResponse
169
+ attr_reader :code, :headers
170
+
171
+ def initialize(code = 200, body = nil, headers = {})
172
+ @code = code
173
+ @body = body
174
+ @headers = headers
175
+ end
176
+
177
+ def body
178
+ @body.to_json
179
+ end
180
+ end
181
+
167
182
  describe "connection management" do
168
183
  before(:each) { subject.start }
169
184
  context "with only one URL in the list" do
@@ -175,8 +190,17 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
175
190
  end
176
191
 
177
192
  context "with multiple URLs in the list" do
193
+ let(:version_ok) do
194
+ MockResponse.new(200, {"tagline" => "You Know, for Search",
195
+ "version" => {
196
+ "number" => '7.13.0',
197
+ "build_flavor" => 'default'}
198
+ })
199
+ end
200
+
178
201
  before :each do
179
202
  allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
203
+ allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
180
204
  end
181
205
  let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
182
206
 
@@ -220,8 +244,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
220
244
  ::LogStash::Util::SafeURI.new("http://otherhost:9201")
221
245
  ] }
222
246
 
247
+ let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
248
+ "version" => {
249
+ "number" => '7.13.0',
250
+ "build_flavor" => 'default'}
251
+ }) }
252
+
223
253
  before(:each) do
224
- allow(subject).to receive(:perform_request_to_url).and_return(nil)
254
+ allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
225
255
  subject.start
226
256
  end
227
257
 
@@ -240,6 +270,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
240
270
  describe "license checking" do
241
271
  before(:each) do
242
272
  allow(subject).to receive(:health_check_request)
273
+ allow(subject).to receive(:elasticsearch?).and_return(true)
243
274
  end
244
275
 
245
276
  let(:options) do
@@ -273,6 +304,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
273
304
 
274
305
  before(:each) do
275
306
  allow(subject).to receive(:health_check_request)
307
+ allow(subject).to receive(:elasticsearch?).and_return(true)
276
308
  end
277
309
 
278
310
  context "if ES doesn't return a valid license" do
@@ -319,3 +351,120 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
319
351
  end
320
352
  end
321
353
  end
354
+
355
+ describe "#elasticsearch?" do
356
+ let(:logger) { Cabin::Channel.get }
357
+ let(:adapter) { double("Manticore Adapter") }
358
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
359
+ let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
360
+ let(:es_node_versions) { [ "0.0.0" ] }
361
+ let(:license_status) { 'active' }
362
+
363
+ subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
364
+
365
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
366
+
367
+ context "in case HTTP error code" do
368
+ it "should fail for 401" do
369
+ allow(adapter).to receive(:perform_request)
370
+ .with(anything, :get, "/", anything, anything)
371
+ .and_return(MockResponse.new(401))
372
+
373
+ expect(subject.elasticsearch?(url)).to be false
374
+ end
375
+
376
+ it "should fail for 403" do
377
+ allow(adapter).to receive(:perform_request)
378
+ .with(anything, :get, "/", anything, anything)
379
+ .and_return(status: 403)
380
+ expect(subject.elasticsearch?(url)).to be false
381
+ end
382
+ end
383
+
384
+ context "when connecting to a cluster which reply without 'version' field" do
385
+ it "should fail" do
386
+ allow(adapter).to receive(:perform_request)
387
+ .with(anything, :get, "/", anything, anything)
388
+ .and_return(body: {"field" => "funky.com"}.to_json)
389
+ expect(subject.elasticsearch?(url)).to be false
390
+ end
391
+ end
392
+
393
+ context "when connecting to a cluster with version < 6.0.0" do
394
+ it "should fail" do
395
+ allow(adapter).to receive(:perform_request)
396
+ .with(anything, :get, "/", anything, anything)
397
+ .and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
398
+ expect(subject.elasticsearch?(url)).to be false
399
+ end
400
+ end
401
+
402
+ context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
403
+ it "must be successful with valid 'tagline'" do
404
+ allow(adapter).to receive(:perform_request)
405
+ .with(anything, :get, "/", anything, anything)
406
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
407
+ expect(subject.elasticsearch?(url)).to be true
408
+ end
409
+
410
+ it "should fail if invalid 'tagline'" do
411
+ allow(adapter).to receive(:perform_request)
412
+ .with(anything, :get, "/", anything, anything)
413
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
414
+ expect(subject.elasticsearch?(url)).to be false
415
+ end
416
+
417
+ it "should fail if 'tagline' is not present" do
418
+ allow(adapter).to receive(:perform_request)
419
+ .with(anything, :get, "/", anything, anything)
420
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
421
+ expect(subject.elasticsearch?(url)).to be false
422
+ end
423
+ end
424
+
425
+ context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
426
+ it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
427
+ allow(adapter).to receive(:perform_request)
428
+ .with(anything, :get, "/", anything, anything)
429
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
430
+ expect(subject.elasticsearch?(url)).to be true
431
+ end
432
+
433
+ it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
434
+ allow(adapter).to receive(:perform_request)
435
+ .with(anything, :get, "/", anything, anything)
436
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
437
+ expect(subject.elasticsearch?(url)).to be false
438
+ end
439
+
440
+ it "should fail if 'build_flavor' is not present and tagline is correct" do
441
+ allow(adapter).to receive(:perform_request)
442
+ .with(anything, :get, "/", anything, anything)
443
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
444
+ expect(subject.elasticsearch?(url)).to be false
445
+ end
446
+ end
447
+
448
+ context "when connecting to a cluster with version >= 7.14.0" do
449
+ it "should fail if 'X-elastic-product' header is not present" do
450
+ allow(adapter).to receive(:perform_request)
451
+ .with(anything, :get, "/", anything, anything)
452
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
453
+ expect(subject.elasticsearch?(url)).to be false
454
+ end
455
+
456
+ it "should fail if 'X-elastic-product' header is present but with bad value" do
457
+ allow(adapter).to receive(:perform_request)
458
+ .with(anything, :get, "/", anything, anything)
459
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
460
+ expect(subject.elasticsearch?(url)).to be false
461
+ end
462
+
463
+ it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
464
+ allow(adapter).to receive(:perform_request)
465
+ .with(anything, :get, "/", anything, anything)
466
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
467
+ expect(subject.elasticsearch?(url)).to be true
468
+ end
469
+ end
470
+ end
@@ -1,6 +1,8 @@
1
1
  require_relative "../../../../spec/spec_helper"
2
2
  require "logstash/outputs/elasticsearch/http_client"
3
3
  require "cabin"
4
+ require "webrick"
5
+ require "java"
4
6
 
5
7
  describe LogStash::Outputs::ElasticSearch::HttpClient do
6
8
  let(:ssl) { nil }
@@ -287,4 +289,71 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
287
289
  end
288
290
  end
289
291
  end
292
+
293
+ class StoppableServer
294
+
295
+ attr_reader :port
296
+
297
+ def initialize()
298
+ queue = Queue.new
299
+ @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
300
+ @first_request = nil
301
+
302
+ @t = java.lang.Thread.new(
303
+ proc do
304
+ begin
305
+ @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
306
+ :Logger => Cabin::Channel.get, # silence WEBrick logging
307
+ :StartCallback => Proc.new {
308
+ queue.push("started")
309
+ }
310
+ @port = @server.config[:Port]
311
+ @server.mount_proc '/headers_check' do |req, res|
312
+ res.body = 'Hello, world from WEBrick mocking server!'
313
+ @first_request = req
314
+ @first_req_waiter.countDown()
315
+ end
316
+
317
+ @server.start
318
+ rescue => e
319
+ puts "Error in webserver thread #{e}"
320
+ # ignore
321
+ end
322
+ end
323
+ )
324
+ @t.daemon = true
325
+ @t.start
326
+ queue.pop # blocks until the server is up
327
+ end
328
+
329
+ def stop
330
+ @server.shutdown
331
+ end
332
+
333
+ def wait_receive_request
334
+ @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
335
+ @first_request
336
+ end
337
+ end
338
+
339
+ describe "#build_adapter" do
340
+ let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options) }
341
+ let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
342
+
343
+ after :each do
344
+ webserver.stop
345
+ end
346
+
347
+ context "the 'user-agent' header" do
348
+ it "contains the Logstash environment details" do
349
+ adapter = client.build_adapter(client.options)
350
+ adapter.perform_request(::LogStash::Util::SafeURI.new("http://localhost:#{webserver.port}"), :get, "/headers_check")
351
+
352
+ request = webserver.wait_receive_request
353
+
354
+ transmitted_user_agent = request.header['user-agent'][0]
355
+ expect(transmitted_user_agent).to match(/Logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-output-elasticsearch\/\d*\.\d*\.\d*/)
356
+ end
357
+ end
358
+ end
290
359
  end
@@ -156,7 +156,7 @@ describe LogStash::Outputs::ElasticSearch do
156
156
  include_examples("an authenticated config")
157
157
  end
158
158
 
159
- context 'claud_auth also set' do
159
+ context 'cloud_auth also set' do
160
160
  let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
161
161
  let(:options) { { "user" => user, "password" => password, "cloud_auth" => "elastic:my-passwd-00" } }
162
162
 
@@ -934,6 +934,7 @@ describe LogStash::Outputs::ElasticSearch do
934
934
  allow(subject).to receive(:last_es_version).and_return es_version
935
935
  # make successful_connection? return true:
936
936
  allow(subject).to receive(:maximum_seen_major_version).and_return Integer(es_version.split('.').first)
937
+ allow(subject).to receive(:alive_urls_count).and_return Integer(1)
937
938
  allow(subject).to receive(:stop_after_successful_connection_thread)
938
939
  end
939
940
 
@@ -12,6 +12,7 @@ describe "whitelisting error types in expected behavior" do
12
12
  before :each do
13
13
  allow(subject.logger).to receive(:warn)
14
14
  allow(subject).to receive(:maximum_seen_major_version).and_return(0)
15
+ allow(subject).to receive(:alive_urls_count).and_return(1)
15
16
  allow(subject).to receive(:finish_register)
16
17
 
17
18
  subject.register
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.0.3
4
+ version: 11.2.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-08-20 00:00:00.000000000 Z
11
+ date: 2021-10-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -140,6 +140,34 @@ dependencies:
140
140
  - - "~>"
141
141
  - !ruby/object:Gem::Version
142
142
  version: '0.6'
143
+ - !ruby/object:Gem::Dependency
144
+ requirement: !ruby/object:Gem::Requirement
145
+ requirements:
146
+ - - ">="
147
+ - !ruby/object:Gem::Version
148
+ version: '0'
149
+ name: webrick
150
+ prerelease: false
151
+ type: :development
152
+ version_requirements: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: '0'
157
+ - !ruby/object:Gem::Dependency
158
+ requirement: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - ">="
161
+ - !ruby/object:Gem::Version
162
+ version: '0'
163
+ name: webmock
164
+ prerelease: false
165
+ type: :development
166
+ version_requirements: !ruby/object:Gem::Requirement
167
+ requirements:
168
+ - - ">="
169
+ - !ruby/object:Gem::Version
170
+ version: '0'
143
171
  - !ruby/object:Gem::Dependency
144
172
  requirement: !ruby/object:Gem::Requirement
145
173
  requirements:
@@ -184,7 +212,6 @@ files:
184
212
  - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
185
213
  - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
186
214
  - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
187
- - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
188
215
  - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
189
216
  - lib/logstash/plugin_mixins/elasticsearch/common.rb
190
217
  - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb
@@ -1 +0,0 @@
1
- lib/logstash/outputs/elasticsearch/templates/ecs-v1/Users/yaauie/src/elastic/logstash-plugins/logsta