logstash-output-elasticsearch 5.4.0-java → 5.4.1-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d4c13cc28976cdad32da8417651ef69ad5695333
4
- data.tar.gz: 145160a4c63cd206a3d869f044b434a169fe5f43
3
+ metadata.gz: d76c66e378b2ccf47fd7cbf0358369eeddc13f9f
4
+ data.tar.gz: c91f5e96604d6eccaff69da784e234baec2d8f05
5
5
  SHA512:
6
- metadata.gz: 0256fab32a89ddafee3dbe18023c4d1bc415d32d1aab8bd30e11bac473760bc6c5fa9beab01b058ebee9e14ec4163f9ef1882527e5a5205592add85b63007b25
7
- data.tar.gz: 297813ad0a6c5103c04fc9cd5085829bb8982bb56be9c15526561fc361d5094f2dbecbde3d5c9dd93b5d4fc1513c1be51dd6ddf8d71c73c15026e802c2411d46
6
+ metadata.gz: 79370d08703d8cd92360d1487b5bcd4cdfde5927abc9f8b7525569d298059c7c10f0f165942d923e2ca7e63f88a3409ee1366ec3a60fa3b8adf261081f0f5e6e
7
+ data.tar.gz: 8bb54f67ecd81847ebc3a296f5249ed70add26b466016fde9cb869962ca0f28916b74cd76278317a2b7c45725cddd64bc4c0346094bda5d6d90a13a3c2f30bf4
data/CHANGELOG.md CHANGED
@@ -1,5 +1,9 @@
1
+ ## 5.4.1
2
+ - Correctly sniff against ES 5.x clusters
3
+
1
4
  ## 5.4.0
2
5
  - Perform healthcheck against hosts right after startup / sniffing
6
+ - Add support for custom query parameters
3
7
 
4
8
  ## 5.3.5
5
9
  - Docs: Remove mention of using the elasticsearch_java output plugin because it is no longer supported
@@ -151,15 +151,48 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
151
151
  ES2_SNIFF_RE_URL = /([^\/]*)?\/?([^:]*):([0-9]+)/
152
152
  # Sniffs and returns the results. Does not update internal URLs!
153
153
  def check_sniff
154
- url, resp = perform_request(:get, '_nodes')
154
+ url, resp = perform_request(:get, '_nodes/http')
155
155
  parsed = LogStash::Json.load(resp.body)
156
- parsed['nodes'].map do |id,info|
156
+
157
+ nodes = parsed['nodes']
158
+ if !nodes || nodes.empty?
159
+ @logger.warn("Sniff returned no nodes! Will not update hosts.")
160
+ return nil
161
+ else
162
+ case major_version(nodes)
163
+ when 5
164
+ sniff_5x(nodes)
165
+ when 2
166
+ sniff_2x_1x(nodes)
167
+ when 1
168
+ sniff_2x_1x(nodes)
169
+ else
170
+ @logger.warn("Could not determine version for nodes in ES cluster!")
171
+ return nil
172
+ end
173
+ end
174
+ end
175
+
176
+ def major_version(nodes)
177
+ k,v = nodes.first; v['version'].split('.').first.to_i
178
+ end
179
+
180
+ def sniff_5x(nodes)
181
+ nodes.map do |id,info|
182
+ if info["http"]
183
+ uri = LogStash::Util::SafeURI.new(info["http"]["publish_address"])
184
+ end
185
+ end.compact
186
+ end
187
+
188
+ def sniff_2x_1x(nodes)
189
+ nodes.map do |id,info|
157
190
  # TODO Make sure this works with shield. Does that listed
158
191
  # stuff as 'https_address?'
192
+
159
193
  addr_str = info['http_address'].to_s
160
194
  next unless addr_str # Skip hosts with HTTP disabled
161
195
 
162
-
163
196
  # Only connect to nodes that serve data
164
197
  # this will skip connecting to client, tribe, and master only nodes
165
198
  # Note that if 'attributes' is NOT set, then that's just a regular node
@@ -171,7 +204,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
171
204
  if matches
172
205
  host = matches[1].empty? ? matches[2] : matches[1]
173
206
  port = matches[3]
174
- URI.parse("#{@scheme}://#{host}:#{port}")
207
+ LogStash::Util::SafeURI.new("#{host}:#{port}")
175
208
  end
176
209
  end.compact
177
210
  end
@@ -237,7 +270,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
237
270
  end
238
271
 
239
272
  def normalize_url(uri)
240
- raise ArgumentError, "Only URI objects may be passed in!" unless uri.is_a?(URI)
273
+ raise ArgumentError, "Only URI/SafeURI objects may be passed in!" unless uri.is_a?(URI) || uri.is_a?(LogStash::Util::SafeURI)
241
274
  uri = uri.clone
242
275
 
243
276
  # Set credentials if need be
@@ -252,6 +285,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
252
285
  end
253
286
 
254
287
  def update_urls(new_urls)
288
+ return if new_urls.nil?
289
+
255
290
  # Normalize URLs
256
291
  new_urls = new_urls.map(&method(:normalize_url))
257
292
 
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-elasticsearch'
4
- s.version = '5.4.0'
4
+ s.version = '5.4.1'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Elasticsearch"
7
7
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -0,0 +1,40 @@
1
+ require "logstash/devutils/rspec/spec_helper"
2
+ require_relative "../../../spec/es_spec_helper"
3
+ require "logstash/outputs/elasticsearch/http_client"
4
+ require "json"
5
+
6
+ describe "pool sniffer", :integration => true do
7
+ let(:logger) { Cabin::Channel.get }
8
+ let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
9
+ let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
10
+ let(:options) { {:resurrect_delay => 2} } # Shorten the delay a bit to speed up tests
11
+
12
+ subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
13
+
14
+ shared_examples("sniff parsing") do |check_exact|
15
+ it "should execute a sniff without error" do
16
+ expect { subject.check_sniff }.not_to raise_error
17
+ end
18
+
19
+ it "should return the correct sniff URL list" do
20
+ uris = subject.check_sniff
21
+
22
+ # ES 1.x returned the public hostname by default. This is hard to approximate
23
+ # so for ES1.x we don't check the *exact* hostname
24
+ if check_exact
25
+ expect(uris).to include(::LogStash::Util::SafeURI.new("//#{get_host_port}"))
26
+ else
27
+ expect(uris.size).to eq(1)
28
+ end
29
+ end
30
+ end
31
+
32
+ describe("Simple sniff parsing") do
33
+ include_examples("sniff parsing", false)
34
+ end
35
+
36
+ # We do a more thorough check on these versions because we can more reliably guess the ip
37
+ describe("Complex sniff parsing ES 5x/2x", :version_greater_than_equal_to_2x => true) do
38
+ include_examples("sniff parsing", true)
39
+ end
40
+ end
@@ -43,31 +43,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
43
43
  expect(subject.sniffer_alive?).to eql(true)
44
44
  end
45
45
  end
46
-
47
- describe "check sniff" do
48
- context "with a good sniff result" do
49
- let(:sniff_resp_path) { File.dirname(__FILE__) + '/../../../../fixtures/5x_node_resp.json' }
50
- let(:sniff_resp) { double("resp") }
51
- let(:sniff_resp_body) { File.open(sniff_resp_path).read }
52
-
53
- before do
54
- allow(subject).to receive(:perform_request).
55
- with(:get, '_nodes').
56
- and_return([double('url'), sniff_resp])
57
- allow(sniff_resp).to receive(:body).and_return(sniff_resp_body)
58
- end
59
-
60
- it "should execute a sniff without error" do
61
- expect { subject.check_sniff }.not_to raise_error
62
- end
63
-
64
- it "should return the correct sniff URL list" do
65
- url_strs = subject.check_sniff.map(&:to_s)
66
- expect(url_strs).to include("http://127.0.0.1:9200")
67
- expect(url_strs).to include("http://127.0.0.1:9201")
68
- end
69
- end
70
- end
71
46
  end
72
47
 
73
48
  describe "closing" do
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.4.0
4
+ version: 5.4.1
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-24 00:00:00.000000000 Z
11
+ date: 2016-12-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -193,7 +193,6 @@ files:
193
193
  - lib/logstash/outputs/elasticsearch/template_manager.rb
194
194
  - logstash-output-elasticsearch.gemspec
195
195
  - spec/es_spec_helper.rb
196
- - spec/fixtures/5x_node_resp.json
197
196
  - spec/fixtures/scripts/scripted_update.groovy
198
197
  - spec/fixtures/scripts/scripted_update_nested.groovy
199
198
  - spec/fixtures/scripts/scripted_upsert.groovy
@@ -204,6 +203,7 @@ files:
204
203
  - spec/integration/outputs/retry_spec.rb
205
204
  - spec/integration/outputs/routing_spec.rb
206
205
  - spec/integration/outputs/secure_spec.rb
206
+ - spec/integration/outputs/sniffer_spec.rb
207
207
  - spec/integration/outputs/templates_5x_spec.rb
208
208
  - spec/integration/outputs/templates_spec.rb
209
209
  - spec/integration/outputs/update_spec.rb
@@ -245,7 +245,6 @@ specification_version: 4
245
245
  summary: Logstash Output to Elasticsearch
246
246
  test_files:
247
247
  - spec/es_spec_helper.rb
248
- - spec/fixtures/5x_node_resp.json
249
248
  - spec/fixtures/scripts/scripted_update.groovy
250
249
  - spec/fixtures/scripts/scripted_update_nested.groovy
251
250
  - spec/fixtures/scripts/scripted_upsert.groovy
@@ -256,6 +255,7 @@ test_files:
256
255
  - spec/integration/outputs/retry_spec.rb
257
256
  - spec/integration/outputs/routing_spec.rb
258
257
  - spec/integration/outputs/secure_spec.rb
258
+ - spec/integration/outputs/sniffer_spec.rb
259
259
  - spec/integration/outputs/templates_5x_spec.rb
260
260
  - spec/integration/outputs/templates_spec.rb
261
261
  - spec/integration/outputs/update_spec.rb
@@ -1,2 +0,0 @@
1
-
2
- {"cluster_name":"elasticsearch","nodes":{"vW8V2o_KSnOa-i97FACWFw":{"name":"Blockbuster","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9200","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Blockbuster"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19048,"mlockall":false},"jvm":{"pid":19048,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781724873,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9300","[::1]:9300","127.0.0.1:9300"],"publish_address":"127.0.0.1:9300","profiles":{}},"http":{"bound_address":["[fe80::1]:9200","[::1]:9200","127.0.0.1:9200"],"publish_address":"127.0.0.1:9200","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}},"BIAIepXSTYufETY06CUpYw":{"name":"Spyne","transport_address":"127.0.0.1:9301","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9201","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Spyne"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19029,"mlockall":false},"jvm":{"pid":19029,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781692985,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9301","[::1]:9301","127.0.0.1:9301"],"publish_address":"127.0.0.1:9301","profiles":{}},"http":{"bound_address":["[fe80::1]:9201","[::1]:9201","127.0.0.1:9201"],"publish_address":"127.0.0.1:9201","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}}}}