logstash-output-elasticsearch 7.4.0-java → 7.4.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +3 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +6 -3
- data/lib/logstash/outputs/elasticsearch/http_client.rb +7 -9
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +31 -12
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +4 -6
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/es_spec_helper.rb +6 -0
- data/spec/integration/outputs/compressed_indexing_spec.rb +46 -44
- data/spec/integration/outputs/delete_spec.rb +51 -49
- data/spec/integration/outputs/groovy_update_spec.rb +131 -129
- data/spec/integration/outputs/index_version_spec.rb +82 -81
- data/spec/integration/outputs/ingest_pipeline_spec.rb +51 -49
- data/spec/integration/outputs/painless_update_spec.rb +132 -130
- data/spec/integration/outputs/parent_spec.rb +56 -54
- data/spec/integration/outputs/sniffer_spec.rb +5 -2
- data/spec/integration/outputs/templates_5x_spec.rb +81 -82
- data/spec/integration/outputs/templates_spec.rb +81 -81
- data/spec/integration/outputs/update_spec.rb +101 -99
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +3 -0
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +1 -1
- data/spec/unit/outputs/elasticsearch/template_manager_spec.rb +13 -25
- metadata +4 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a5f3f952de8b5aa9f1e43b98f6a8cf7b8b0eca6f
|
4
|
+
data.tar.gz: bd133b48b12f09f9193ad8fae93bde676bbe77b1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 51737449d12145f393943545566650d037a6558ca5a4eb1e8d64249f416213c4c046f3fc8455ccd22d9727345e8c51fb5f97b050c875b58ec72cac3542589637
|
7
|
+
data.tar.gz: f567270ae2fc471f624276823f3e70425b30a084bdf92ec348e2ede49a562b80d8fc2a71022f83296458917908c5069b5c7f7fe71deffe21ff822225f65be09e
|
data/CHANGELOG.md
CHANGED
@@ -18,7 +18,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
18
18
|
def register
|
19
19
|
@stopping = Concurrent::AtomicBoolean.new(false)
|
20
20
|
# To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
|
21
|
-
@dlq_writer =
|
21
|
+
@dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
|
22
22
|
|
23
23
|
setup_hosts # properly sets @hosts
|
24
24
|
build_client
|
@@ -279,8 +279,11 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
279
279
|
end
|
280
280
|
end
|
281
281
|
|
282
|
-
def
|
283
|
-
|
282
|
+
def dlq_enabled?
|
283
|
+
# TODO there should be a better way to query if DLQ is enabled
|
284
|
+
# See more in: https://github.com/elastic/logstash/issues/8064
|
285
|
+
respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
|
286
|
+
!execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
|
284
287
|
end
|
285
288
|
end
|
286
289
|
end; end; end
|
@@ -82,9 +82,8 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
82
82
|
template_put(name, template)
|
83
83
|
end
|
84
84
|
|
85
|
-
def
|
86
|
-
|
87
|
-
LogStash::Json.load(response.body)["version"]
|
85
|
+
def connected_es_versions
|
86
|
+
@pool.connected_es_versions
|
88
87
|
end
|
89
88
|
|
90
89
|
def bulk(actions)
|
@@ -135,7 +134,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
135
134
|
def bulk_send(body_stream)
|
136
135
|
params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
|
137
136
|
# Discard the URL
|
138
|
-
|
137
|
+
response = @pool.post(@bulk_path, params, body_stream.string)
|
139
138
|
if !body_stream.closed?
|
140
139
|
body_stream.truncate(0)
|
141
140
|
body_stream.seek(0)
|
@@ -151,12 +150,12 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
151
150
|
end
|
152
151
|
|
153
152
|
def get(path)
|
154
|
-
|
153
|
+
response = @pool.get(path, nil)
|
155
154
|
LogStash::Json.load(response.body)
|
156
155
|
end
|
157
156
|
|
158
157
|
def post(path, params = {}, body_string)
|
159
|
-
|
158
|
+
response = @pool.post(path, params, body_string)
|
160
159
|
LogStash::Json.load(response.body)
|
161
160
|
end
|
162
161
|
|
@@ -327,15 +326,14 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
327
326
|
end
|
328
327
|
|
329
328
|
def template_exists?(name)
|
330
|
-
|
329
|
+
response = @pool.head("/_template/#{name}")
|
331
330
|
response.code >= 200 && response.code <= 299
|
332
331
|
end
|
333
332
|
|
334
333
|
def template_put(name, template)
|
335
334
|
path = "_template/#{name}"
|
336
335
|
logger.info("Installing elasticsearch template to #{path}")
|
337
|
-
|
338
|
-
response
|
336
|
+
@pool.put(path, nil, LogStash::Json.dump(template))
|
339
337
|
end
|
340
338
|
|
341
339
|
# Build a bulk item for an elasticsearch update action
|
@@ -107,6 +107,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
107
107
|
@state_mutex.synchronize { @url_info }
|
108
108
|
end
|
109
109
|
|
110
|
+
def connected_es_versions
|
111
|
+
@state_mutex.synchronize do
|
112
|
+
@url_info.values.select {|v| v[:state] == :alive }.map {|v| v[:version] }
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
110
116
|
def urls
|
111
117
|
url_info.keys
|
112
118
|
end
|
@@ -154,7 +160,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
154
160
|
ES2_SNIFF_RE_URL = /([^\/]*)?\/?([^:]*):([0-9]+)/
|
155
161
|
# Sniffs and returns the results. Does not update internal URLs!
|
156
162
|
def check_sniff
|
157
|
-
_, resp = perform_request(:get, @sniffing_path)
|
163
|
+
_, url_meta, resp = perform_request(:get, @sniffing_path)
|
158
164
|
parsed = LogStash::Json.load(resp.body)
|
159
165
|
|
160
166
|
nodes = parsed['nodes']
|
@@ -162,12 +168,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
162
168
|
@logger.warn("Sniff returned no nodes! Will not update hosts.")
|
163
169
|
return nil
|
164
170
|
else
|
165
|
-
case major_version(
|
171
|
+
case major_version(url_meta[:version])
|
166
172
|
when 5, 6
|
167
173
|
sniff_5x_and_above(nodes)
|
168
|
-
when 2
|
169
|
-
sniff_2x_1x(nodes)
|
170
|
-
when 1
|
174
|
+
when 2, 1
|
171
175
|
sniff_2x_1x(nodes)
|
172
176
|
else
|
173
177
|
@logger.warn("Could not determine version for nodes in ES cluster!")
|
@@ -176,8 +180,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
176
180
|
end
|
177
181
|
end
|
178
182
|
|
179
|
-
def major_version(
|
180
|
-
|
183
|
+
def major_version(version_string)
|
184
|
+
version_string.split('.').first.to_i
|
181
185
|
end
|
182
186
|
|
183
187
|
def sniff_5x_and_above(nodes)
|
@@ -237,7 +241,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
237
241
|
response = perform_request_to_url(url, :head, @healthcheck_path)
|
238
242
|
# If no exception was raised it must have succeeded!
|
239
243
|
logger.warn("Restored connection to ES instance", :url => url.sanitized.to_s)
|
240
|
-
|
244
|
+
# We reconnected to this node, check its ES version
|
245
|
+
es_version = get_es_version(url)
|
246
|
+
@state_mutex.synchronize do
|
247
|
+
meta[:version] = es_version
|
248
|
+
meta[:state] = :alive
|
249
|
+
end
|
241
250
|
rescue HostUnreachableError, BadResponseCodeError => e
|
242
251
|
logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: url.sanitized.to_s, error_type: e.class, error: e.message)
|
243
252
|
end
|
@@ -253,15 +262,16 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
253
262
|
end
|
254
263
|
|
255
264
|
def perform_request(method, path, params={}, body=nil)
|
256
|
-
with_connection do |url|
|
265
|
+
with_connection do |url, url_meta|
|
257
266
|
resp = perform_request_to_url(url, method, path, params, body)
|
258
|
-
[url, resp]
|
267
|
+
[url, url_meta, resp]
|
259
268
|
end
|
260
269
|
end
|
261
270
|
|
262
271
|
[:get, :put, :post, :delete, :patch, :head].each do |method|
|
263
272
|
define_method(method) do |path, params={}, body=nil|
|
264
|
-
perform_request(method, path, params, body)
|
273
|
+
_, _, response = perform_request(method, path, params, body)
|
274
|
+
response
|
265
275
|
end
|
266
276
|
end
|
267
277
|
|
@@ -323,6 +333,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
323
333
|
@state_mutex.synchronize { @url_info.size }
|
324
334
|
end
|
325
335
|
|
336
|
+
def es_versions
|
337
|
+
@state_mutex.synchronize { @url_info.size }
|
338
|
+
end
|
339
|
+
|
326
340
|
def add_url(url)
|
327
341
|
@url_info[url] ||= empty_url_meta
|
328
342
|
end
|
@@ -344,7 +358,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
344
358
|
# Custom error class used here so that users may retry attempts if they receive this error
|
345
359
|
# should they choose to
|
346
360
|
raise NoConnectionAvailableError, "No Available connections" unless url
|
347
|
-
yield url
|
361
|
+
yield url, url_meta
|
348
362
|
rescue HostUnreachableError => e
|
349
363
|
# Mark the connection as dead here since this is likely not transient
|
350
364
|
mark_dead(url, e)
|
@@ -415,5 +429,10 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
415
429
|
end
|
416
430
|
end
|
417
431
|
end
|
432
|
+
|
433
|
+
def get_es_version(url)
|
434
|
+
request = perform_request_to_url(url, :get, ROOT_URI_PATH)
|
435
|
+
LogStash::Json.load(request.body)["version"]["number"]
|
436
|
+
end
|
418
437
|
end
|
419
438
|
end; end; end; end;
|
@@ -12,12 +12,10 @@ module LogStash; module Outputs; class ElasticSearch
|
|
12
12
|
end
|
13
13
|
|
14
14
|
private
|
15
|
-
def self.get_es_version(client)
|
16
|
-
client.get_version
|
17
|
-
end
|
18
|
-
|
19
15
|
def self.get_es_major_version(client)
|
20
|
-
|
16
|
+
# get the elasticsearch version of each node in the pool and
|
17
|
+
# pick the biggest major version
|
18
|
+
client.connected_es_versions.uniq.map {|version| version.split(".").first.to_i}.max
|
21
19
|
end
|
22
20
|
|
23
21
|
def self.get_template(path, es_major_version)
|
@@ -30,7 +28,7 @@ module LogStash; module Outputs; class ElasticSearch
|
|
30
28
|
end
|
31
29
|
|
32
30
|
def self.default_template_path(es_major_version)
|
33
|
-
template_version = es_major_version ==
|
31
|
+
template_version = es_major_version == 1 ? 2 : es_major_version
|
34
32
|
default_template_name = "elasticsearch-template-es#{template_version}x.json"
|
35
33
|
::File.expand_path(default_template_name, ::File.dirname(__FILE__))
|
36
34
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-elasticsearch'
|
3
|
-
s.version = '7.4.
|
3
|
+
s.version = '7.4.1'
|
4
4
|
s.licenses = ['apache-2.0']
|
5
5
|
s.summary = "Logstash Output to Elasticsearch"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/es_spec_helper.rb
CHANGED
@@ -10,6 +10,12 @@ module ESHelper
|
|
10
10
|
def get_client
|
11
11
|
Elasticsearch::Client.new(:hosts => [get_host_port])
|
12
12
|
end
|
13
|
+
|
14
|
+
def self.es_version_satisfies?(requirement)
|
15
|
+
es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
|
16
|
+
es_release_version = Gem::Version.new(es_version).release
|
17
|
+
Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
|
18
|
+
end
|
13
19
|
end
|
14
20
|
|
15
21
|
RSpec.configure do |config|
|
@@ -9,58 +9,60 @@ RSpec::Matchers.define :a_valid_gzip_encoded_string do
|
|
9
9
|
}
|
10
10
|
end
|
11
11
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
{
|
20
|
-
|
21
|
-
|
22
|
-
|
12
|
+
if ESHelper.es_version_satisfies?(">= 5")
|
13
|
+
describe "indexing with http_compression turned on", :integration => true do
|
14
|
+
let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
|
15
|
+
let(:index) { 10.times.collect { rand(10).to_s }.join("") }
|
16
|
+
let(:type) { 10.times.collect { rand(10).to_s }.join("") }
|
17
|
+
let(:event_count) { 10000 + rand(500) }
|
18
|
+
let(:events) { event_count.times.map { event }.to_a }
|
19
|
+
let(:config) {
|
20
|
+
{
|
21
|
+
"hosts" => get_host_port,
|
22
|
+
"index" => index,
|
23
|
+
"http_compression" => true
|
24
|
+
}
|
23
25
|
}
|
24
|
-
|
25
|
-
subject { LogStash::Outputs::ElasticSearch.new(config) }
|
26
|
+
subject { LogStash::Outputs::ElasticSearch.new(config) }
|
26
27
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
28
|
+
let(:es_url) { "http://#{get_host_port}" }
|
29
|
+
let(:index_url) {"#{es_url}/#{index}"}
|
30
|
+
let(:http_client_options) { {} }
|
31
|
+
let(:http_client) do
|
32
|
+
Manticore::Client.new(http_client_options)
|
33
|
+
end
|
33
34
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
shared_examples "an indexer" do
|
39
|
-
it "ships events" do
|
40
|
-
subject.multi_receive(events)
|
35
|
+
before do
|
36
|
+
subject.register
|
37
|
+
end
|
41
38
|
|
42
|
-
|
39
|
+
shared_examples "an indexer" do
|
40
|
+
it "ships events" do
|
41
|
+
subject.multi_receive(events)
|
43
42
|
|
44
|
-
|
45
|
-
result = LogStash::Json.load(response.body)
|
46
|
-
cur_count = result["count"]
|
47
|
-
expect(cur_count).to eq(event_count)
|
43
|
+
http_client.post("#{es_url}/_refresh").call
|
48
44
|
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
expect(
|
53
|
-
|
45
|
+
response = http_client.get("#{index_url}/_count?q=*")
|
46
|
+
result = LogStash::Json.load(response.body)
|
47
|
+
cur_count = result["count"]
|
48
|
+
expect(cur_count).to eq(event_count)
|
49
|
+
|
50
|
+
response = http_client.get("#{index_url}/_search?q=*&size=1000")
|
51
|
+
result = LogStash::Json.load(response.body)
|
52
|
+
result["hits"]["hits"].each do |doc|
|
53
|
+
expect(doc["_type"]).to eq(type)
|
54
|
+
expect(doc["_index"]).to eq(index)
|
55
|
+
end
|
54
56
|
end
|
55
57
|
end
|
56
|
-
end
|
57
58
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
59
|
+
it "sets the correct content-encoding header and body is compressed" do
|
60
|
+
expect(subject.client.pool.adapter.client).to receive(:send).
|
61
|
+
with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
|
62
|
+
and_call_original
|
63
|
+
subject.multi_receive(events)
|
64
|
+
end
|
64
65
|
|
65
|
-
|
66
|
+
it_behaves_like("an indexer")
|
67
|
+
end
|
66
68
|
end
|
@@ -2,62 +2,64 @@ require_relative "../../../spec/es_spec_helper"
|
|
2
2
|
require "logstash/outputs/elasticsearch"
|
3
3
|
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
let(:es) { get_client }
|
9
|
-
|
10
|
-
before :each do
|
11
|
-
# Delete all templates first.
|
12
|
-
# Clean ES of data before we start.
|
13
|
-
es.indices.delete_template(:name => "*")
|
14
|
-
# This can fail if there are no indexes, ignore failure.
|
15
|
-
es.indices.delete(:index => "*") rescue nil
|
16
|
-
es.indices.refresh
|
17
|
-
end
|
5
|
+
if ESHelper.es_version_satisfies?(">= 2")
|
6
|
+
describe "Versioned delete", :integration => true do
|
7
|
+
require "logstash/outputs/elasticsearch"
|
18
8
|
|
19
|
-
|
20
|
-
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|
9
|
+
let(:es) { get_client }
|
21
10
|
|
22
|
-
before do
|
23
|
-
|
11
|
+
before :each do
|
12
|
+
# Delete all templates first.
|
13
|
+
# Clean ES of data before we start.
|
14
|
+
es.indices.delete_template(:name => "*")
|
15
|
+
# This can fail if there are no indexes, ignore failure.
|
16
|
+
es.indices.delete(:index => "*") rescue nil
|
17
|
+
es.indices.refresh
|
24
18
|
end
|
25
19
|
|
26
|
-
|
27
|
-
{
|
28
|
-
"manage_template" => true,
|
29
|
-
"index" => "logstash-delete",
|
30
|
-
"template_overwrite" => true,
|
31
|
-
"hosts" => get_host_port(),
|
32
|
-
"document_id" => "%{my_id}",
|
33
|
-
"version" => "%{my_version}",
|
34
|
-
"version_type" => "external",
|
35
|
-
"action" => "%{my_action}"
|
36
|
-
}
|
37
|
-
end
|
20
|
+
context "when delete only" do
|
21
|
+
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|
38
22
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
23
|
+
before do
|
24
|
+
subject.register
|
25
|
+
end
|
26
|
+
|
27
|
+
let(:settings) do
|
28
|
+
{
|
29
|
+
"manage_template" => true,
|
30
|
+
"index" => "logstash-delete",
|
31
|
+
"template_overwrite" => true,
|
32
|
+
"hosts" => get_host_port(),
|
33
|
+
"document_id" => "%{my_id}",
|
34
|
+
"version" => "%{my_version}",
|
35
|
+
"version_type" => "external",
|
36
|
+
"action" => "%{my_action}"
|
37
|
+
}
|
38
|
+
end
|
39
|
+
|
40
|
+
it "should ignore non-monotonic external version updates" do
|
41
|
+
id = "ev2"
|
42
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
|
43
|
+
r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
|
44
|
+
expect(r['_version']).to eq(99)
|
45
|
+
expect(r['_source']['message']).to eq('foo')
|
46
|
+
|
47
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
|
48
|
+
r2 = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
|
49
|
+
expect(r2['_version']).to eq(99)
|
50
|
+
expect(r2['_source']['message']).to eq('foo')
|
51
|
+
end
|
51
52
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
53
|
+
it "should commit monotonic external version updates" do
|
54
|
+
id = "ev3"
|
55
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
|
56
|
+
r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
|
57
|
+
expect(r['_version']).to eq(99)
|
58
|
+
expect(r['_source']['message']).to eq('foo')
|
58
59
|
|
59
|
-
|
60
|
-
|
60
|
+
subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
|
61
|
+
expect { es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
|
62
|
+
end
|
61
63
|
end
|
62
64
|
end
|
63
65
|
end
|