logstash-output-elasticsearch 11.0.5-java → 11.2.2-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-elasticsearch'
3
- s.version = '11.0.5'
3
+ s.version = '11.2.2'
4
4
 
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Stores logs in Elasticsearch"
@@ -30,6 +30,8 @@ Gem::Specification.new do |s|
30
30
  s.add_development_dependency 'logstash-devutils'
31
31
  s.add_development_dependency 'flores'
32
32
  s.add_development_dependency 'cabin', ['~> 0.6']
33
+ s.add_development_dependency 'webrick'
34
+ s.add_development_dependency 'webmock'
33
35
  # Still used in some specs, we should remove this ASAP
34
36
  s.add_development_dependency 'elasticsearch'
35
37
  end
@@ -80,7 +80,7 @@ module ESHelper
80
80
  end
81
81
 
82
82
  def self.es_version_satisfies?(*requirement)
83
- es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
83
+ es_version = nilify(RSpec.configuration.filter[:es_version]) || nilify(ENV['ES_VERSION']) || nilify(ENV['ELASTIC_STACK_VERSION'])
84
84
  if es_version.nil?
85
85
  puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
86
86
  return false
@@ -89,6 +89,15 @@ module ESHelper
89
89
  Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
90
90
  end
91
91
 
92
+ private
93
+ def self.nilify(str)
94
+ if str.nil?
95
+ return str
96
+ end
97
+ str.empty? ? nil : str
98
+ end
99
+
100
+ public
92
101
  def clean(client)
93
102
  client.indices.delete_template(:name => "*")
94
103
  client.indices.delete_index_template(:name => "logstash*") rescue nil
@@ -50,15 +50,18 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
50
50
 
51
51
  describe "healthcheck url handling" do
52
52
  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
53
+ before(:example) do
54
+ expect(adapter).to receive(:perform_request).with(anything, :get, "/", anything, anything) do |url, _, _, _, _|
55
+ expect(url.path).to be_empty
56
+ end
57
+ end
53
58
 
54
59
  context "and not setting healthcheck_path" do
55
60
  it "performs the healthcheck to the root" do
56
- expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
57
- expect(method).to eq(:head)
61
+ expect(adapter).to receive(:perform_request).with(anything, :head, "/", anything, anything) do |url, _, _, _, _|
58
62
  expect(url.path).to be_empty
59
- expect(req_path).to eq("/")
60
63
  end
61
- subject.healthcheck!
64
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
62
65
  end
63
66
  end
64
67
 
@@ -66,12 +69,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
66
69
  let(:healthcheck_path) { "/my/health" }
67
70
  let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
68
71
  it "performs the healthcheck to the healthcheck_path" do
69
- expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
70
- expect(method).to eq(:head)
72
+ expect(adapter).to receive(:perform_request).with(anything, :head, eq(healthcheck_path), anything, anything) do |url, _, _, _, _|
71
73
  expect(url.path).to be_empty
72
- expect(req_path).to eq(healthcheck_path)
73
74
  end
74
- subject.healthcheck!
75
+ expect { subject.healthcheck! }.to raise_error(LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch")
75
76
  end
76
77
  end
77
78
  end
@@ -164,6 +165,20 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
164
165
  end
165
166
  end
166
167
 
168
+ class MockResponse
169
+ attr_reader :code, :headers
170
+
171
+ def initialize(code = 200, body = nil, headers = {})
172
+ @code = code
173
+ @body = body
174
+ @headers = headers
175
+ end
176
+
177
+ def body
178
+ @body.to_json
179
+ end
180
+ end
181
+
167
182
  describe "connection management" do
168
183
  before(:each) { subject.start }
169
184
  context "with only one URL in the list" do
@@ -175,8 +190,17 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
175
190
  end
176
191
 
177
192
  context "with multiple URLs in the list" do
193
+ let(:version_ok) do
194
+ MockResponse.new(200, {"tagline" => "You Know, for Search",
195
+ "version" => {
196
+ "number" => '7.13.0',
197
+ "build_flavor" => 'default'}
198
+ })
199
+ end
200
+
178
201
  before :each do
179
202
  allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
203
+ allow(adapter).to receive(:perform_request).with(anything, :get, subject.healthcheck_path, {}, nil).and_return(version_ok)
180
204
  end
181
205
  let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
182
206
 
@@ -220,8 +244,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
220
244
  ::LogStash::Util::SafeURI.new("http://otherhost:9201")
221
245
  ] }
222
246
 
247
+ let(:valid_response) { MockResponse.new(200, {"tagline" => "You Know, for Search",
248
+ "version" => {
249
+ "number" => '7.13.0',
250
+ "build_flavor" => 'default'}
251
+ }) }
252
+
223
253
  before(:each) do
224
- allow(subject).to receive(:perform_request_to_url).and_return(nil)
254
+ allow(subject).to receive(:perform_request_to_url).and_return(valid_response)
225
255
  subject.start
226
256
  end
227
257
 
@@ -240,6 +270,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
240
270
  describe "license checking" do
241
271
  before(:each) do
242
272
  allow(subject).to receive(:health_check_request)
273
+ allow(subject).to receive(:elasticsearch?).and_return(true)
243
274
  end
244
275
 
245
276
  let(:options) do
@@ -273,6 +304,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
273
304
 
274
305
  before(:each) do
275
306
  allow(subject).to receive(:health_check_request)
307
+ allow(subject).to receive(:elasticsearch?).and_return(true)
276
308
  end
277
309
 
278
310
  context "if ES doesn't return a valid license" do
@@ -319,3 +351,120 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
319
351
  end
320
352
  end
321
353
  end
354
+
355
+ describe "#elasticsearch?" do
356
+ let(:logger) { Cabin::Channel.get }
357
+ let(:adapter) { double("Manticore Adapter") }
358
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
359
+ let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
360
+ let(:es_node_versions) { [ "0.0.0" ] }
361
+ let(:license_status) { 'active' }
362
+
363
+ subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
364
+
365
+ let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
366
+
367
+ context "in case HTTP error code" do
368
+ it "should fail for 401" do
369
+ allow(adapter).to receive(:perform_request)
370
+ .with(anything, :get, "/", anything, anything)
371
+ .and_return(MockResponse.new(401))
372
+
373
+ expect(subject.elasticsearch?(url)).to be false
374
+ end
375
+
376
+ it "should fail for 403" do
377
+ allow(adapter).to receive(:perform_request)
378
+ .with(anything, :get, "/", anything, anything)
379
+ .and_return(status: 403)
380
+ expect(subject.elasticsearch?(url)).to be false
381
+ end
382
+ end
383
+
384
+ context "when connecting to a cluster which reply without 'version' field" do
385
+ it "should fail" do
386
+ allow(adapter).to receive(:perform_request)
387
+ .with(anything, :get, "/", anything, anything)
388
+ .and_return(body: {"field" => "funky.com"}.to_json)
389
+ expect(subject.elasticsearch?(url)).to be false
390
+ end
391
+ end
392
+
393
+ context "when connecting to a cluster with version < 6.0.0" do
394
+ it "should fail" do
395
+ allow(adapter).to receive(:perform_request)
396
+ .with(anything, :get, "/", anything, anything)
397
+ .and_return(200, {"version" => { "number" => "5.0.0"}}.to_json)
398
+ expect(subject.elasticsearch?(url)).to be false
399
+ end
400
+ end
401
+
402
+ context "when connecting to a cluster with version in [6.0.0..7.0.0)" do
403
+ it "must be successful with valid 'tagline'" do
404
+ allow(adapter).to receive(:perform_request)
405
+ .with(anything, :get, "/", anything, anything)
406
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You Know, for Search"}))
407
+ expect(subject.elasticsearch?(url)).to be true
408
+ end
409
+
410
+ it "should fail if invalid 'tagline'" do
411
+ allow(adapter).to receive(:perform_request)
412
+ .with(anything, :get, "/", anything, anything)
413
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}, "tagline" => "You don't know"}))
414
+ expect(subject.elasticsearch?(url)).to be false
415
+ end
416
+
417
+ it "should fail if 'tagline' is not present" do
418
+ allow(adapter).to receive(:perform_request)
419
+ .with(anything, :get, "/", anything, anything)
420
+ .and_return(MockResponse.new(200, {"version" => {"number" => "6.5.0"}}))
421
+ expect(subject.elasticsearch?(url)).to be false
422
+ end
423
+ end
424
+
425
+ context "when connecting to a cluster with version in [7.0.0..7.14.0)" do
426
+ it "must be successful is 'build_flavor' is 'default' and tagline is correct" do
427
+ allow(adapter).to receive(:perform_request)
428
+ .with(anything, :get, "/", anything, anything)
429
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "default"}, "tagline": "You Know, for Search"}))
430
+ expect(subject.elasticsearch?(url)).to be true
431
+ end
432
+
433
+ it "should fail if 'build_flavor' is not 'default' and tagline is correct" do
434
+ allow(adapter).to receive(:perform_request)
435
+ .with(anything, :get, "/", anything, anything)
436
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0", "build_flavor": "oss"}, "tagline": "You Know, for Search"}))
437
+ expect(subject.elasticsearch?(url)).to be false
438
+ end
439
+
440
+ it "should fail if 'build_flavor' is not present and tagline is correct" do
441
+ allow(adapter).to receive(:perform_request)
442
+ .with(anything, :get, "/", anything, anything)
443
+ .and_return(MockResponse.new(200, {"version": {"number": "7.5.0"}, "tagline": "You Know, for Search"}))
444
+ expect(subject.elasticsearch?(url)).to be false
445
+ end
446
+ end
447
+
448
+ context "when connecting to a cluster with version >= 7.14.0" do
449
+ it "should fail if 'X-elastic-product' header is not present" do
450
+ allow(adapter).to receive(:perform_request)
451
+ .with(anything, :get, "/", anything, anything)
452
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}))
453
+ expect(subject.elasticsearch?(url)).to be false
454
+ end
455
+
456
+ it "should fail if 'X-elastic-product' header is present but with bad value" do
457
+ allow(adapter).to receive(:perform_request)
458
+ .with(anything, :get, "/", anything, anything)
459
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'not good'}))
460
+ expect(subject.elasticsearch?(url)).to be false
461
+ end
462
+
463
+ it "must be successful when 'X-elastic-product' header is present with 'Elasticsearch' value" do
464
+ allow(adapter).to receive(:perform_request)
465
+ .with(anything, :get, "/", anything, anything)
466
+ .and_return(MockResponse.new(200, {"version": {"number": "7.14.0"}}, {'X-elastic-product' => 'Elasticsearch'}))
467
+ expect(subject.elasticsearch?(url)).to be true
468
+ end
469
+ end
470
+ end
@@ -1,6 +1,8 @@
1
1
  require_relative "../../../../spec/spec_helper"
2
2
  require "logstash/outputs/elasticsearch/http_client"
3
3
  require "cabin"
4
+ require "webrick"
5
+ require "java"
4
6
 
5
7
  describe LogStash::Outputs::ElasticSearch::HttpClient do
6
8
  let(:ssl) { nil }
@@ -287,4 +289,71 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
287
289
  end
288
290
  end
289
291
  end
292
+
293
+ class StoppableServer
294
+
295
+ attr_reader :port
296
+
297
+ def initialize()
298
+ queue = Queue.new
299
+ @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
300
+ @first_request = nil
301
+
302
+ @t = java.lang.Thread.new(
303
+ proc do
304
+ begin
305
+ @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
306
+ :Logger => Cabin::Channel.get, # silence WEBrick logging
307
+ :StartCallback => Proc.new {
308
+ queue.push("started")
309
+ }
310
+ @port = @server.config[:Port]
311
+ @server.mount_proc '/headers_check' do |req, res|
312
+ res.body = 'Hello, world from WEBrick mocking server!'
313
+ @first_request = req
314
+ @first_req_waiter.countDown()
315
+ end
316
+
317
+ @server.start
318
+ rescue => e
319
+ puts "Error in webserver thread #{e}"
320
+ # ignore
321
+ end
322
+ end
323
+ )
324
+ @t.daemon = true
325
+ @t.start
326
+ queue.pop # blocks until the server is up
327
+ end
328
+
329
+ def stop
330
+ @server.shutdown
331
+ end
332
+
333
+ def wait_receive_request
334
+ @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
335
+ @first_request
336
+ end
337
+ end
338
+
339
+ describe "#build_adapter" do
340
+ let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options) }
341
+ let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
342
+
343
+ after :each do
344
+ webserver.stop
345
+ end
346
+
347
+ context "the 'user-agent' header" do
348
+ it "contains the Logstash environment details" do
349
+ adapter = client.build_adapter(client.options)
350
+ adapter.perform_request(::LogStash::Util::SafeURI.new("http://localhost:#{webserver.port}"), :get, "/headers_check")
351
+
352
+ request = webserver.wait_receive_request
353
+
354
+ transmitted_user_agent = request.header['user-agent'][0]
355
+ expect(transmitted_user_agent).to match(/Logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-output-elasticsearch\/\d*\.\d*\.\d*/)
356
+ end
357
+ end
358
+ end
290
359
  end
@@ -156,7 +156,7 @@ describe LogStash::Outputs::ElasticSearch do
156
156
  include_examples("an authenticated config")
157
157
  end
158
158
 
159
- context 'claud_auth also set' do
159
+ context 'cloud_auth also set' do
160
160
  let(:do_register) { false } # this is what we want to test, so we disable the before(:each) call
161
161
  let(:options) { { "user" => user, "password" => password, "cloud_auth" => "elastic:my-passwd-00" } }
162
162
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.0.5
4
+ version: 11.2.2
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-08-30 00:00:00.000000000 Z
11
+ date: 2021-11-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -140,6 +140,34 @@ dependencies:
140
140
  - - "~>"
141
141
  - !ruby/object:Gem::Version
142
142
  version: '0.6'
143
+ - !ruby/object:Gem::Dependency
144
+ requirement: !ruby/object:Gem::Requirement
145
+ requirements:
146
+ - - ">="
147
+ - !ruby/object:Gem::Version
148
+ version: '0'
149
+ name: webrick
150
+ prerelease: false
151
+ type: :development
152
+ version_requirements: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: '0'
157
+ - !ruby/object:Gem::Dependency
158
+ requirement: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - ">="
161
+ - !ruby/object:Gem::Version
162
+ version: '0'
163
+ name: webmock
164
+ prerelease: false
165
+ type: :development
166
+ version_requirements: !ruby/object:Gem::Requirement
167
+ requirements:
168
+ - - ">="
169
+ - !ruby/object:Gem::Version
170
+ version: '0'
143
171
  - !ruby/object:Gem::Dependency
144
172
  requirement: !ruby/object:Gem::Requirement
145
173
  requirements:
@@ -184,7 +212,6 @@ files:
184
212
  - lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json
185
213
  - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-6x.json
186
214
  - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-7x.json
187
- - lib/logstash/outputs/elasticsearch/templates/ecs-v1/elasticsearch-8x.json
188
215
  - lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
189
216
  - lib/logstash/plugin_mixins/elasticsearch/common.rb
190
217
  - lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb
@@ -1 +0,0 @@
1
- lib/logstash/outputs/elasticsearch/templates/ecs-v1/Users/yaauie/src/elastic/logstash-plugins/logsta