logstash-output-elasticsearch 5.4.0-java → 5.4.1-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +40 -5
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/integration/outputs/sniffer_spec.rb +40 -0
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +0 -25
- metadata +4 -4
- data/spec/fixtures/5x_node_resp.json +0 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d76c66e378b2ccf47fd7cbf0358369eeddc13f9f
|
4
|
+
data.tar.gz: c91f5e96604d6eccaff69da784e234baec2d8f05
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 79370d08703d8cd92360d1487b5bcd4cdfde5927abc9f8b7525569d298059c7c10f0f165942d923e2ca7e63f88a3409ee1366ec3a60fa3b8adf261081f0f5e6e
|
7
|
+
data.tar.gz: 8bb54f67ecd81847ebc3a296f5249ed70add26b466016fde9cb869962ca0f28916b74cd76278317a2b7c45725cddd64bc4c0346094bda5d6d90a13a3c2f30bf4
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
|
+
## 5.4.1
|
2
|
+
- Correctly sniff against ES 5.x clusters
|
3
|
+
|
1
4
|
## 5.4.0
|
2
5
|
- Perform healthcheck against hosts right after startup / sniffing
|
6
|
+
- Add support for custom query parameters
|
3
7
|
|
4
8
|
## 5.3.5
|
5
9
|
- Docs: Remove mention of using the elasticsearch_java output plugin because it is no longer supported
|
@@ -151,15 +151,48 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
151
151
|
ES2_SNIFF_RE_URL = /([^\/]*)?\/?([^:]*):([0-9]+)/
|
152
152
|
# Sniffs and returns the results. Does not update internal URLs!
|
153
153
|
def check_sniff
|
154
|
-
url, resp = perform_request(:get, '_nodes')
|
154
|
+
url, resp = perform_request(:get, '_nodes/http')
|
155
155
|
parsed = LogStash::Json.load(resp.body)
|
156
|
-
|
156
|
+
|
157
|
+
nodes = parsed['nodes']
|
158
|
+
if !nodes || nodes.empty?
|
159
|
+
@logger.warn("Sniff returned no nodes! Will not update hosts.")
|
160
|
+
return nil
|
161
|
+
else
|
162
|
+
case major_version(nodes)
|
163
|
+
when 5
|
164
|
+
sniff_5x(nodes)
|
165
|
+
when 2
|
166
|
+
sniff_2x_1x(nodes)
|
167
|
+
when 1
|
168
|
+
sniff_2x_1x(nodes)
|
169
|
+
else
|
170
|
+
@logger.warn("Could not determine version for nodes in ES cluster!")
|
171
|
+
return nil
|
172
|
+
end
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
def major_version(nodes)
|
177
|
+
k,v = nodes.first; v['version'].split('.').first.to_i
|
178
|
+
end
|
179
|
+
|
180
|
+
def sniff_5x(nodes)
|
181
|
+
nodes.map do |id,info|
|
182
|
+
if info["http"]
|
183
|
+
uri = LogStash::Util::SafeURI.new(info["http"]["publish_address"])
|
184
|
+
end
|
185
|
+
end.compact
|
186
|
+
end
|
187
|
+
|
188
|
+
def sniff_2x_1x(nodes)
|
189
|
+
nodes.map do |id,info|
|
157
190
|
# TODO Make sure this works with shield. Does that listed
|
158
191
|
# stuff as 'https_address?'
|
192
|
+
|
159
193
|
addr_str = info['http_address'].to_s
|
160
194
|
next unless addr_str # Skip hosts with HTTP disabled
|
161
195
|
|
162
|
-
|
163
196
|
# Only connect to nodes that serve data
|
164
197
|
# this will skip connecting to client, tribe, and master only nodes
|
165
198
|
# Note that if 'attributes' is NOT set, then that's just a regular node
|
@@ -171,7 +204,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
171
204
|
if matches
|
172
205
|
host = matches[1].empty? ? matches[2] : matches[1]
|
173
206
|
port = matches[3]
|
174
|
-
|
207
|
+
LogStash::Util::SafeURI.new("#{host}:#{port}")
|
175
208
|
end
|
176
209
|
end.compact
|
177
210
|
end
|
@@ -237,7 +270,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
237
270
|
end
|
238
271
|
|
239
272
|
def normalize_url(uri)
|
240
|
-
raise ArgumentError, "Only URI objects may be passed in!" unless uri.is_a?(URI)
|
273
|
+
raise ArgumentError, "Only URI/SafeURI objects may be passed in!" unless uri.is_a?(URI) || uri.is_a?(LogStash::Util::SafeURI)
|
241
274
|
uri = uri.clone
|
242
275
|
|
243
276
|
# Set credentials if need be
|
@@ -252,6 +285,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
252
285
|
end
|
253
286
|
|
254
287
|
def update_urls(new_urls)
|
288
|
+
return if new_urls.nil?
|
289
|
+
|
255
290
|
# Normalize URLs
|
256
291
|
new_urls = new_urls.map(&method(:normalize_url))
|
257
292
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-elasticsearch'
|
4
|
-
s.version = '5.4.
|
4
|
+
s.version = '5.4.1'
|
5
5
|
s.licenses = ['apache-2.0']
|
6
6
|
s.summary = "Logstash Output to Elasticsearch"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require "logstash/devutils/rspec/spec_helper"
|
2
|
+
require_relative "../../../spec/es_spec_helper"
|
3
|
+
require "logstash/outputs/elasticsearch/http_client"
|
4
|
+
require "json"
|
5
|
+
|
6
|
+
describe "pool sniffer", :integration => true do
|
7
|
+
let(:logger) { Cabin::Channel.get }
|
8
|
+
let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
|
9
|
+
let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
|
10
|
+
let(:options) { {:resurrect_delay => 2} } # Shorten the delay a bit to speed up tests
|
11
|
+
|
12
|
+
subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
|
13
|
+
|
14
|
+
shared_examples("sniff parsing") do |check_exact|
|
15
|
+
it "should execute a sniff without error" do
|
16
|
+
expect { subject.check_sniff }.not_to raise_error
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should return the correct sniff URL list" do
|
20
|
+
uris = subject.check_sniff
|
21
|
+
|
22
|
+
# ES 1.x returned the public hostname by default. This is hard to approximate
|
23
|
+
# so for ES1.x we don't check the *exact* hostname
|
24
|
+
if check_exact
|
25
|
+
expect(uris).to include(::LogStash::Util::SafeURI.new("//#{get_host_port}"))
|
26
|
+
else
|
27
|
+
expect(uris.size).to eq(1)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
describe("Simple sniff parsing") do
|
33
|
+
include_examples("sniff parsing", false)
|
34
|
+
end
|
35
|
+
|
36
|
+
# We do a more thorough check on these versions because we can more reliably guess the ip
|
37
|
+
describe("Complex sniff parsing ES 5x/2x", :version_greater_than_equal_to_2x => true) do
|
38
|
+
include_examples("sniff parsing", true)
|
39
|
+
end
|
40
|
+
end
|
@@ -43,31 +43,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
|
|
43
43
|
expect(subject.sniffer_alive?).to eql(true)
|
44
44
|
end
|
45
45
|
end
|
46
|
-
|
47
|
-
describe "check sniff" do
|
48
|
-
context "with a good sniff result" do
|
49
|
-
let(:sniff_resp_path) { File.dirname(__FILE__) + '/../../../../fixtures/5x_node_resp.json' }
|
50
|
-
let(:sniff_resp) { double("resp") }
|
51
|
-
let(:sniff_resp_body) { File.open(sniff_resp_path).read }
|
52
|
-
|
53
|
-
before do
|
54
|
-
allow(subject).to receive(:perform_request).
|
55
|
-
with(:get, '_nodes').
|
56
|
-
and_return([double('url'), sniff_resp])
|
57
|
-
allow(sniff_resp).to receive(:body).and_return(sniff_resp_body)
|
58
|
-
end
|
59
|
-
|
60
|
-
it "should execute a sniff without error" do
|
61
|
-
expect { subject.check_sniff }.not_to raise_error
|
62
|
-
end
|
63
|
-
|
64
|
-
it "should return the correct sniff URL list" do
|
65
|
-
url_strs = subject.check_sniff.map(&:to_s)
|
66
|
-
expect(url_strs).to include("http://127.0.0.1:9200")
|
67
|
-
expect(url_strs).to include("http://127.0.0.1:9201")
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
46
|
end
|
72
47
|
|
73
48
|
describe "closing" do
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.4.
|
4
|
+
version: 5.4.1
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-12-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -193,7 +193,6 @@ files:
|
|
193
193
|
- lib/logstash/outputs/elasticsearch/template_manager.rb
|
194
194
|
- logstash-output-elasticsearch.gemspec
|
195
195
|
- spec/es_spec_helper.rb
|
196
|
-
- spec/fixtures/5x_node_resp.json
|
197
196
|
- spec/fixtures/scripts/scripted_update.groovy
|
198
197
|
- spec/fixtures/scripts/scripted_update_nested.groovy
|
199
198
|
- spec/fixtures/scripts/scripted_upsert.groovy
|
@@ -204,6 +203,7 @@ files:
|
|
204
203
|
- spec/integration/outputs/retry_spec.rb
|
205
204
|
- spec/integration/outputs/routing_spec.rb
|
206
205
|
- spec/integration/outputs/secure_spec.rb
|
206
|
+
- spec/integration/outputs/sniffer_spec.rb
|
207
207
|
- spec/integration/outputs/templates_5x_spec.rb
|
208
208
|
- spec/integration/outputs/templates_spec.rb
|
209
209
|
- spec/integration/outputs/update_spec.rb
|
@@ -245,7 +245,6 @@ specification_version: 4
|
|
245
245
|
summary: Logstash Output to Elasticsearch
|
246
246
|
test_files:
|
247
247
|
- spec/es_spec_helper.rb
|
248
|
-
- spec/fixtures/5x_node_resp.json
|
249
248
|
- spec/fixtures/scripts/scripted_update.groovy
|
250
249
|
- spec/fixtures/scripts/scripted_update_nested.groovy
|
251
250
|
- spec/fixtures/scripts/scripted_upsert.groovy
|
@@ -256,6 +255,7 @@ test_files:
|
|
256
255
|
- spec/integration/outputs/retry_spec.rb
|
257
256
|
- spec/integration/outputs/routing_spec.rb
|
258
257
|
- spec/integration/outputs/secure_spec.rb
|
258
|
+
- spec/integration/outputs/sniffer_spec.rb
|
259
259
|
- spec/integration/outputs/templates_5x_spec.rb
|
260
260
|
- spec/integration/outputs/templates_spec.rb
|
261
261
|
- spec/integration/outputs/update_spec.rb
|
@@ -1,2 +0,0 @@
|
|
1
|
-
|
2
|
-
{"cluster_name":"elasticsearch","nodes":{"vW8V2o_KSnOa-i97FACWFw":{"name":"Blockbuster","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9200","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Blockbuster"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19048,"mlockall":false},"jvm":{"pid":19048,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781724873,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9300","[::1]:9300","127.0.0.1:9300"],"publish_address":"127.0.0.1:9300","profiles":{}},"http":{"bound_address":["[fe80::1]:9200","[::1]:9200","127.0.0.1:9200"],"publish_address":"127.0.0.1:9200","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}},"BIAIepXSTYufETY06CUpYw":{"name":"Spyne","transport_address":"127.0.0.1:9301","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9201","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Spyne"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19029,"mlockall":false},"jvm":{"pid":19029,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781692985,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9301","[::1]:9301","127.0.0.1:9301"],"publish_address":"127.0.0.1:9301","profiles":{}},"http":{"bound_address":["[fe80::1]:9201","[::1]:9201","127.0.0.1:9201"],"publish_address":"127.0.0.1:9201","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}}}}
|