logstash-filter-elasticsearch 3.15.2 → 3.15.3
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7e85d9b1122e7e095666cdc3bfa1dd5fbd3ef62ce5cf15d667ffa92932e5748e
|
4
|
+
data.tar.gz: cbcf76ec5d048bb18f467520f4a646a08e5ea39a8d44ace7a3322e7fea0eee06
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6bc45e8790eede5047013506fe8b9f8ba97ad08365f6ab99d5fd7ca120e051f916301054e23932d068823ba8f0904757de9bc0bb255d13b1967b339c600e4011
|
7
|
+
data.tar.gz: '07842227fb3c356f08012193925fe1b5ae055ead761600c029e0ea4a788b97744113e35ebf97c05e223270e14017bf1d10b66a0ca038a810ca5054c1c2ff8389'
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 3.15.3
|
2
|
+
- Fixes a memory leak that occurs when a pipeline containing this filter terminates, which could become significant if the pipeline is cycled repeatedly [#173](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/173)
|
3
|
+
|
1
4
|
## 3.15.2
|
2
5
|
- Added checking for `query` and `query_template`. [#171](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/171)
|
3
6
|
|
@@ -22,6 +22,9 @@ module LogStash
|
|
22
22
|
transport_options[:headers].merge!(setup_api_key(api_key))
|
23
23
|
transport_options[:headers].merge!({ 'user-agent' => "#{user_agent}" })
|
24
24
|
|
25
|
+
transport_options[:pool_max] = 1000
|
26
|
+
transport_options[:pool_max_per_route] = 100
|
27
|
+
|
25
28
|
logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('')
|
26
29
|
transport_options[:proxy] = proxy.to_s if proxy && !proxy.eql?('')
|
27
30
|
|
@@ -4,6 +4,7 @@ require "logstash/namespace"
|
|
4
4
|
require "logstash/json"
|
5
5
|
require 'logstash/plugin_mixins/ca_trusted_fingerprint_support'
|
6
6
|
require "logstash/plugin_mixins/normalize_config_support"
|
7
|
+
require "monitor"
|
7
8
|
|
8
9
|
require_relative "elasticsearch/client"
|
9
10
|
require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore"
|
@@ -139,7 +140,8 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
139
140
|
|
140
141
|
include LogStash::PluginMixins::NormalizeConfigSupport
|
141
142
|
|
142
|
-
|
143
|
+
include MonitorMixin
|
144
|
+
attr_reader :shared_client
|
143
145
|
|
144
146
|
##
|
145
147
|
# @override to handle proxy => '' as if none was set
|
@@ -159,8 +161,6 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
159
161
|
end
|
160
162
|
|
161
163
|
def register
|
162
|
-
@clients_pool = java.util.concurrent.ConcurrentHashMap.new
|
163
|
-
|
164
164
|
#Load query if it exists
|
165
165
|
if @query_template
|
166
166
|
if File.zero?(@query_template)
|
@@ -352,7 +352,9 @@ class LogStash::Filters::Elasticsearch < LogStash::Filters::Base
|
|
352
352
|
end
|
353
353
|
|
354
354
|
def get_client
|
355
|
-
@
|
355
|
+
@shared_client || synchronize do
|
356
|
+
@shared_client ||= new_client
|
357
|
+
end
|
356
358
|
end
|
357
359
|
|
358
360
|
# get an array of path elements from a path reference
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-elasticsearch'
|
4
|
-
s.version = '3.15.
|
4
|
+
s.version = '3.15.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Copies fields from previous log events in Elasticsearch to current events "
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -91,21 +91,6 @@ describe LogStash::Filters::Elasticsearch do
|
|
91
91
|
Thread.current[:filter_elasticsearch_client] = nil
|
92
92
|
end
|
93
93
|
|
94
|
-
# Since the Elasticsearch Ruby client is not thread safe
|
95
|
-
# and under high load we can get error with the connection pool
|
96
|
-
# we have decided to create a new instance per worker thread which
|
97
|
-
# will be lazy created on the first call to `#filter`
|
98
|
-
#
|
99
|
-
# I am adding a simple test case for future changes
|
100
|
-
it "uses a different connection object per thread wait" do
|
101
|
-
expect(plugin.clients_pool.size).to eq(0)
|
102
|
-
|
103
|
-
Thread.new { plugin.filter(event) }.join
|
104
|
-
Thread.new { plugin.filter(event) }.join
|
105
|
-
|
106
|
-
expect(plugin.clients_pool.size).to eq(2)
|
107
|
-
end
|
108
|
-
|
109
94
|
it "should enhance the current event with new data" do
|
110
95
|
plugin.filter(event)
|
111
96
|
expect(event.get("code")).to eq(404)
|
@@ -466,6 +451,32 @@ describe LogStash::Filters::Elasticsearch do
|
|
466
451
|
Thread.current[:filter_elasticsearch_client] = nil
|
467
452
|
end
|
468
453
|
|
454
|
+
it 'uses a threadsafe transport adapter' do
|
455
|
+
client = plugin.send(:get_client).client
|
456
|
+
# we currently rely on the threadsafety guarantees provided by Manticore
|
457
|
+
# this spec is a safeguard to trigger an assessment of thread-safety should
|
458
|
+
# we choose a different transport adapter in the future.
|
459
|
+
transport_class = extract_transport(client).options.fetch(:transport_class)
|
460
|
+
expect(transport_class).to equal ::Elasticsearch::Transport::Transport::HTTP::Manticore
|
461
|
+
end
|
462
|
+
|
463
|
+
it 'uses a client with sufficient connection pool size' do
|
464
|
+
client = plugin.send(:get_client).client
|
465
|
+
transport_options = extract_transport(client).options.fetch(:transport_options)
|
466
|
+
# pool_max and pool_max_per_route are manticore-specific transport options
|
467
|
+
expect(transport_options).to include(:pool_max => 1000, :pool_max_per_route => 100)
|
468
|
+
end
|
469
|
+
|
470
|
+
it 'uses a single shared client across threads' do
|
471
|
+
q = Queue.new
|
472
|
+
10.times.map do
|
473
|
+
Thread.new(plugin) { |instance| q.push instance.send(:get_client) }
|
474
|
+
end.map(&:join)
|
475
|
+
|
476
|
+
first = q.pop
|
477
|
+
expect(q.pop).to be(first) until q.empty?
|
478
|
+
end
|
479
|
+
|
469
480
|
describe "cloud.id" do
|
470
481
|
let(:valid_cloud_id) do
|
471
482
|
'sample:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA=='
|
@@ -712,9 +723,9 @@ describe LogStash::Filters::Elasticsearch do
|
|
712
723
|
end
|
713
724
|
|
714
725
|
it "should read and send non-ascii query" do
|
715
|
-
expect(client).to receive(:search).with(
|
726
|
+
expect(client).to receive(:search).with({
|
716
727
|
:body => { "query" => { "terms" => { "lock" => [ "잠금", "uzávěr" ] } } },
|
717
|
-
:index => "")
|
728
|
+
:index => ""})
|
718
729
|
|
719
730
|
plugin.filter(LogStash::Event.new)
|
720
731
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.15.
|
4
|
+
version: 3.15.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|