logstash-filter-elasticsearch 3.15.3 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/filters/elasticsearch/client.rb +18 -2
- data/lib/logstash/filters/elasticsearch.rb +16 -7
- data/logstash-filter-elasticsearch.gemspec +2 -2
- data/spec/filters/elasticsearch_spec.rb +97 -4
- data/spec/filters/elasticsearch_ssl_spec.rb +2 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c214ce4f38ac730daec06cafc6c71630b1d6c3c9aacaf2fcbbf3457c055401f1
|
4
|
+
data.tar.gz: 2139bde2dffc838b33ae816325bed9172d78cbb66125f6c204b9e8dd802a9879
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7de34c594ee9d49c567cb70809fc09ebe70f19def9c14e8898a080a7eaac0e889ab0e714a9cae0341bc200e661a5cbf2faca97eb95aab18423387eff24eae88b
|
7
|
+
data.tar.gz: c0cc4308ef74c309a326d47c0c3901811b3b8c0f5f0e3619949cc9b7cc48cb81d078d65391f3020b221f99774a402d90d70d5248094e26f64859248d651a7e61
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 3.16.0
|
2
|
+
- Added request header `Elastic-Api-Version` for serverless [#174](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/174)
|
3
|
+
|
1
4
|
## 3.15.3
|
2
5
|
- Fixes a memory leak that occurs when a pipeline containing this filter terminates, which could become significant if the pipeline is cycled repeatedly [#173](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/173)
|
3
6
|
|
@@ -10,6 +10,9 @@ module LogStash
|
|
10
10
|
|
11
11
|
attr_reader :client
|
12
12
|
|
13
|
+
BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze
|
14
|
+
DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze
|
15
|
+
|
13
16
|
def initialize(logger, hosts, options = {})
|
14
17
|
user = options.fetch(:user, nil)
|
15
18
|
password = options.fetch(:password, nil)
|
@@ -17,7 +20,8 @@ module LogStash
|
|
17
20
|
proxy = options.fetch(:proxy, nil)
|
18
21
|
user_agent = options[:user_agent]
|
19
22
|
|
20
|
-
transport_options = {
|
23
|
+
transport_options = { }
|
24
|
+
transport_options[:headers] = options.fetch(:serverless, false) ? DEFAULT_EAV_HEADER.dup : {}
|
21
25
|
transport_options[:headers].merge!(setup_basic_auth(user, password))
|
22
26
|
transport_options[:headers].merge!(setup_api_key(api_key))
|
23
27
|
transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
|
@@ -46,10 +50,22 @@ module LogStash
|
|
46
50
|
@client = ::Elasticsearch::Client.new(client_options)
|
47
51
|
end
|
48
52
|
|
49
|
-
def search(params)
|
53
|
+
def search(params={})
|
50
54
|
@client.search(params)
|
51
55
|
end
|
52
56
|
|
57
|
+
def info
|
58
|
+
@client.info
|
59
|
+
end
|
60
|
+
|
61
|
+
def build_flavor
|
62
|
+
@build_flavor ||= info&.dig('version', 'build_flavor')
|
63
|
+
end
|
64
|
+
|
65
|
+
def serverless?
|
66
|
+
@is_serverless ||= (build_flavor == BUILD_FLAVOR_SERVERLESS)
|
67
|
+
end
|
68
|
+
|
53
69
|
private
|
54
70
|
|
55
71
|
def setup_hosts(hosts, ssl_enabled)
|
@@ -179,6 +179,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
179
179
|
@hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s
|
180
180
|
|
181
181
|
test_connection!
|
182
|
+
setup_serverless
|
182
183
|
end # def register
|
183
184
|
|
184
185
|
def filter(event)
|
@@ -260,14 +261,15 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
260
261
|
private
|
261
262
|
|
262
263
|
def client_options
|
263
|
-
{
|
264
|
+
@client_options ||= {
|
264
265
|
:user => @user,
|
265
266
|
:password => @password,
|
266
267
|
:api_key => @api_key,
|
267
268
|
:proxy => @proxy,
|
268
269
|
:ssl => client_ssl_options,
|
269
270
|
:retry_on_failure => @retry_on_failure,
|
270
|
-
:retry_on_status => @retry_on_status
|
271
|
+
:retry_on_status => @retry_on_status,
|
272
|
+
:user_agent => prepare_user_agent
|
271
273
|
}
|
272
274
|
end
|
273
275
|
|
@@ -344,11 +346,7 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
344
346
|
def new_client
|
345
347
|
# NOTE: could pass cloud-id/cloud-auth to client but than we would need to be stricter on ES version requirement
|
346
348
|
# and also LS parsing might differ from ES client's parsing so for consistency we do not pass cloud options ...
|
347
|
-
|
348
|
-
|
349
|
-
opts[:user_agent] = prepare_user_agent
|
350
|
-
|
351
|
-
LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, opts)
|
349
|
+
LogStash::Filters::ElasticsearchClient.new(@logger, @hosts, client_options)
|
352
350
|
end
|
353
351
|
|
354
352
|
def get_client
|
@@ -478,6 +476,17 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
478
476
|
end
|
479
477
|
end
|
480
478
|
|
479
|
+
def setup_serverless
|
480
|
+
if get_client.serverless?
|
481
|
+
@client_options[:serverless] = true
|
482
|
+
@shared_client = new_client
|
483
|
+
get_client.info
|
484
|
+
end
|
485
|
+
rescue => e
|
486
|
+
@logger.error("Failed to retrieve Elasticsearch info", message: e.message, exception: e.class, backtrace: e.backtrace)
|
487
|
+
raise LogStash::ConfigurationError, "Could not connect to a compatible version of Elasticsearch"
|
488
|
+
end
|
489
|
+
|
481
490
|
def setup_ssl_params!
|
482
491
|
@ssl_enabled = normalize_config(:ssl_enabled) do |normalize|
|
483
492
|
normalize.with_deprecated_alias(:ssl)
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-elasticsearch'
|
4
|
-
s.version = '3.
|
4
|
+
s.version = '3.16.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Copies fields from previous log events in Elasticsearch to current events "
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
-
s.add_runtime_dependency 'elasticsearch', ">= 7.14.
|
24
|
+
s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5
|
25
25
|
s.add_runtime_dependency 'manticore', ">= 0.7.1"
|
26
26
|
s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0'
|
27
27
|
s.add_runtime_dependency 'logstash-mixin-normalize_config_support', '~>1.0'
|
@@ -22,6 +22,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
22
22
|
|
23
23
|
before do
|
24
24
|
allow(plugin).to receive(:test_connection!)
|
25
|
+
allow(plugin).to receive(:setup_serverless)
|
25
26
|
end
|
26
27
|
|
27
28
|
it "should not raise an exception" do
|
@@ -49,6 +50,26 @@ describe LogStash::Filters::Elasticsearch do
|
|
49
50
|
end
|
50
51
|
end
|
51
52
|
|
53
|
+
context "against serverless Elasticsearch" do
|
54
|
+
let(:config) { { "query" => "*" } }
|
55
|
+
let(:filter_client) { double("filter_client") }
|
56
|
+
let(:es_client) { double("es_client") }
|
57
|
+
|
58
|
+
before do
|
59
|
+
allow(plugin).to receive(:test_connection!)
|
60
|
+
allow(plugin).to receive(:get_client).and_return(filter_client)
|
61
|
+
allow(filter_client).to receive(:serverless?).and_return(true)
|
62
|
+
allow(filter_client).to receive(:client).and_return(es_client)
|
63
|
+
allow(es_client).to receive(:info).with(a_hash_including(:headers => LogStash::Filters::ElasticsearchClient::DEFAULT_EAV_HEADER)).and_raise(
|
64
|
+
Elasticsearch::Transport::Transport::Errors::BadRequest.new
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "raises an exception when Elastic Api Version is not supported" do
|
69
|
+
expect {plugin.register}.to raise_error(LogStash::ConfigurationError)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
52
73
|
context "query settings" do
|
53
74
|
it "raise an exception when query and query_template are empty" do
|
54
75
|
plugin = described_class.new({})
|
@@ -84,6 +105,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
84
105
|
allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
|
85
106
|
allow(client).to receive(:search).and_return(response)
|
86
107
|
allow(plugin).to receive(:test_connection!)
|
108
|
+
allow(plugin).to receive(:setup_serverless)
|
87
109
|
plugin.register
|
88
110
|
end
|
89
111
|
|
@@ -423,7 +445,9 @@ describe LogStash::Filters::Elasticsearch do
|
|
423
445
|
let(:plugin) { described_class.new(config) }
|
424
446
|
let(:event) { LogStash::Event.new({}) }
|
425
447
|
|
426
|
-
|
448
|
+
# elasticsearch-ruby 7.17.9 initialize two user agent headers, `user-agent` and `User-Agent`
|
449
|
+
# hence, fail this header size test case
|
450
|
+
xit "client should sent the expect user-agent" do
|
427
451
|
plugin.register
|
428
452
|
|
429
453
|
request = webserver.wait_receive_request
|
@@ -445,6 +469,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
445
469
|
|
446
470
|
before(:each) do
|
447
471
|
allow(plugin).to receive(:test_connection!)
|
472
|
+
allow(plugin).to receive(:setup_serverless)
|
448
473
|
end
|
449
474
|
|
450
475
|
after(:each) do
|
@@ -619,6 +644,47 @@ describe LogStash::Filters::Elasticsearch do
|
|
619
644
|
end
|
620
645
|
end
|
621
646
|
|
647
|
+
describe "Elastic Api Header" do
|
648
|
+
let(:config) { {"query" => "*"} }
|
649
|
+
let(:plugin) { described_class.new(config) }
|
650
|
+
let(:headers) {{'x-elastic-product' => 'Elasticsearch'}}
|
651
|
+
let(:cluster_info) { {"version" => {"number" => "8.10.0", "build_flavor" => build_flavor}, "tagline" => "You Know, for Search"} }
|
652
|
+
let(:mock_resp) { MockResponse.new(200, cluster_info, headers) }
|
653
|
+
|
654
|
+
before do
|
655
|
+
expect(plugin).to receive(:test_connection!)
|
656
|
+
end
|
657
|
+
|
658
|
+
context "serverless" do
|
659
|
+
let(:build_flavor) { "serverless" }
|
660
|
+
|
661
|
+
before do
|
662
|
+
allow_any_instance_of(Elasticsearch::Client).to receive(:perform_request).with(any_args).and_return(mock_resp)
|
663
|
+
end
|
664
|
+
|
665
|
+
it 'propagates header to es client' do
|
666
|
+
plugin.register
|
667
|
+
client = plugin.send(:get_client).client
|
668
|
+
expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_including("Elastic-Api-Version" => "2023-10-31")
|
669
|
+
end
|
670
|
+
end
|
671
|
+
|
672
|
+
context "stateful" do
|
673
|
+
let(:build_flavor) { "default" }
|
674
|
+
|
675
|
+
before do
|
676
|
+
expect_any_instance_of(Elasticsearch::Client).to receive(:perform_request).with(any_args).and_return(mock_resp)
|
677
|
+
end
|
678
|
+
|
679
|
+
it 'does not propagate header to es client' do
|
680
|
+
plugin.register
|
681
|
+
client = plugin.send(:get_client).client
|
682
|
+
expect( extract_transport(client).options[:transport_options][:headers] ).to match hash_not_including("Elastic-Api-Version" => "2023-10-31")
|
683
|
+
end
|
684
|
+
end
|
685
|
+
|
686
|
+
end
|
687
|
+
|
622
688
|
describe "ca_trusted_fingerprint" do
|
623
689
|
let(:ca_trusted_fingerprint) { SecureRandom.hex(32) }
|
624
690
|
let(:config) { {"ssl_enabled" => true, "ca_trusted_fingerprint" => ca_trusted_fingerprint, "query" => "*"}}
|
@@ -627,7 +693,10 @@ describe LogStash::Filters::Elasticsearch do
|
|
627
693
|
|
628
694
|
if Gem::Version.create(LOGSTASH_VERSION) >= Gem::Version.create("8.3.0")
|
629
695
|
context 'the generated trust_strategy' do
|
630
|
-
before(:each)
|
696
|
+
before(:each) do
|
697
|
+
allow(plugin).to receive(:test_connection!)
|
698
|
+
allow(plugin).to receive(:setup_serverless)
|
699
|
+
end
|
631
700
|
|
632
701
|
it 'is passed to the Manticore client' do
|
633
702
|
expect(Manticore::Client).to receive(:new)
|
@@ -666,7 +735,10 @@ describe LogStash::Filters::Elasticsearch do
|
|
666
735
|
|
667
736
|
subject(:plugin) { described_class.new(config) }
|
668
737
|
|
669
|
-
before(:each)
|
738
|
+
before(:each) do
|
739
|
+
allow(plugin).to receive(:test_connection!)
|
740
|
+
allow(plugin).to receive(:setup_serverless)
|
741
|
+
end
|
670
742
|
|
671
743
|
it 'is passed to the Manticore client' do
|
672
744
|
expect(Manticore::Client).to receive(:new)
|
@@ -694,7 +766,10 @@ describe LogStash::Filters::Elasticsearch do
|
|
694
766
|
let(:config) { {"query" => "*"} }
|
695
767
|
let(:plugin) { described_class.new(config) }
|
696
768
|
|
697
|
-
before
|
769
|
+
before do
|
770
|
+
allow(plugin).to receive(:test_connection!)
|
771
|
+
allow(plugin).to receive(:setup_serverless)
|
772
|
+
end
|
698
773
|
|
699
774
|
it "should set localhost:9200 as hosts" do
|
700
775
|
plugin.register
|
@@ -719,6 +794,7 @@ describe LogStash::Filters::Elasticsearch do
|
|
719
794
|
before(:each) do
|
720
795
|
allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client)
|
721
796
|
allow(plugin).to receive(:test_connection!)
|
797
|
+
allow(plugin).to receive(:setup_serverless)
|
722
798
|
plugin.register
|
723
799
|
end
|
724
800
|
|
@@ -736,4 +812,21 @@ describe LogStash::Filters::Elasticsearch do
|
|
736
812
|
client.transport.respond_to?(:transport) ? client.transport.transport : client.transport
|
737
813
|
end
|
738
814
|
|
815
|
+
class MockResponse
|
816
|
+
attr_reader :code, :headers
|
817
|
+
|
818
|
+
def initialize(code = 200, body = nil, headers = {})
|
819
|
+
@code = code
|
820
|
+
@body = body
|
821
|
+
@headers = headers
|
822
|
+
end
|
823
|
+
|
824
|
+
def body
|
825
|
+
@body
|
826
|
+
end
|
827
|
+
|
828
|
+
def status
|
829
|
+
@code
|
830
|
+
end
|
831
|
+
end
|
739
832
|
end
|
@@ -15,6 +15,8 @@ describe "SSL options" do
|
|
15
15
|
before do
|
16
16
|
allow(es_client_double).to receive(:close)
|
17
17
|
allow(es_client_double).to receive(:ping).with(any_args).and_return(double("pong").as_null_object)
|
18
|
+
allow(es_client_double).to receive(:info).with(any_args).and_return({"version" => {"number" => "7.5.0", "build_flavor" => "default"},
|
19
|
+
"tagline" => "You Know, for Search"})
|
18
20
|
allow(Elasticsearch::Client).to receive(:new).and_return(es_client_double)
|
19
21
|
end
|
20
22
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.16.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-09-
|
11
|
+
date: 2023-09-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -35,7 +35,7 @@ dependencies:
|
|
35
35
|
requirements:
|
36
36
|
- - ">="
|
37
37
|
- !ruby/object:Gem::Version
|
38
|
-
version: 7.14.
|
38
|
+
version: 7.14.9
|
39
39
|
name: elasticsearch
|
40
40
|
prerelease: false
|
41
41
|
type: :runtime
|
@@ -43,7 +43,7 @@ dependencies:
|
|
43
43
|
requirements:
|
44
44
|
- - ">="
|
45
45
|
- !ruby/object:Gem::Version
|
46
|
-
version: 7.14.
|
46
|
+
version: 7.14.9
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
49
49
|
requirements:
|