logstash-output-elasticsearch 5.4.0-java → 5.4.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4c13cc28976cdad32da8417651ef69ad5695333
-  data.tar.gz: 145160a4c63cd206a3d869f044b434a169fe5f43
+  metadata.gz: d76c66e378b2ccf47fd7cbf0358369eeddc13f9f
+  data.tar.gz: c91f5e96604d6eccaff69da784e234baec2d8f05
 SHA512:
-  metadata.gz: 0256fab32a89ddafee3dbe18023c4d1bc415d32d1aab8bd30e11bac473760bc6c5fa9beab01b058ebee9e14ec4163f9ef1882527e5a5205592add85b63007b25
-  data.tar.gz: 297813ad0a6c5103c04fc9cd5085829bb8982bb56be9c15526561fc361d5094f2dbecbde3d5c9dd93b5d4fc1513c1be51dd6ddf8d71c73c15026e802c2411d46
+  metadata.gz: 79370d08703d8cd92360d1487b5bcd4cdfde5927abc9f8b7525569d298059c7c10f0f165942d923e2ca7e63f88a3409ee1366ec3a60fa3b8adf261081f0f5e6e
+  data.tar.gz: 8bb54f67ecd81847ebc3a296f5249ed70add26b466016fde9cb869962ca0f28916b74cd76278317a2b7c45725cddd64bc4c0346094bda5d6d90a13a3c2f30bf4

data/CHANGELOG.md

@@ -1,5 +1,9 @@
+## 5.4.1
+- Correctly sniff against ES 5.x clusters
+
 ## 5.4.0
 - Perform healthcheck against hosts right after startup / sniffing
+- Add support for custom query parameters
 
 ## 5.3.5
 - Docs: Remove mention of using the elasticsearch_java output plugin because it is no longer supported

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -151,15 +151,48 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     ES2_SNIFF_RE_URL  = /([^\/]*)?\/?([^:]*):([0-9]+)/
     # Sniffs and returns the results. Does not update internal URLs!
     def check_sniff
-      url, resp = perform_request(:get, '_nodes')
+      url, resp = perform_request(:get, '_nodes/http')
       parsed = LogStash::Json.load(resp.body)
-      parsed['nodes'].map do |id,info|
+      
+      nodes = parsed['nodes']
+      if !nodes || nodes.empty?
+        @logger.warn("Sniff returned no nodes! Will not update hosts.")
+        return nil
+      else
+        case major_version(nodes)
+        when 5
+          sniff_5x(nodes)
+        when 2
+          sniff_2x_1x(nodes)
+        when 1
+          sniff_2x_1x(nodes)
+        else
+          @logger.warn("Could not determine version for nodes in ES cluster!")
+          return nil
+        end
+      end
+    end
+    
+    def major_version(nodes)
+      k,v = nodes.first; v['version'].split('.').first.to_i
+    end
+    
+    def sniff_5x(nodes)
+      nodes.map do |id,info|
+        if info["http"]
+          uri = LogStash::Util::SafeURI.new(info["http"]["publish_address"])
+        end
+      end.compact
+    end
+    
+    def sniff_2x_1x(nodes)
+      nodes.map do |id,info|
         # TODO Make sure this works with shield. Does that listed
         # stuff as 'https_address?'
+        
         addr_str = info['http_address'].to_s
         next unless addr_str # Skip hosts with HTTP disabled
 
-
         # Only connect to nodes that serve data
         # this will skip connecting to client, tribe, and master only nodes
         # Note that if 'attributes' is NOT set, then that's just a regular node
@@ -171,7 +204,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         if matches
           host = matches[1].empty? ? matches[2] : matches[1]
           port = matches[3]
-          URI.parse("#{@scheme}://#{host}:#{port}")
+          LogStash::Util::SafeURI.new("#{host}:#{port}")
         end
       end.compact
     end
@@ -237,7 +270,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def normalize_url(uri)
-      raise ArgumentError, "Only URI objects may be passed in!" unless uri.is_a?(URI)
+      raise ArgumentError, "Only URI/SafeURI objects may be passed in!" unless uri.is_a?(URI) || uri.is_a?(LogStash::Util::SafeURI)
       uri = uri.clone
 
       # Set credentials if need be
@@ -252,6 +285,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def update_urls(new_urls)
+      return if new_urls.nil?
+      
       # Normalize URLs
       new_urls = new_urls.map(&method(:normalize_url))
 

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '5.4.0'
+  s.version         = '5.4.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/sniffer_spec.rb

@@ -0,0 +1,40 @@
+require "logstash/devutils/rspec/spec_helper"
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "json"
+
+describe "pool sniffer", :integration => true do
+  let(:logger) { Cabin::Channel.get }
+  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
+  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
+  let(:options) { {:resurrect_delay => 2} } # Shorten the delay a bit to speed up tests
+
+  subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
+  
+  shared_examples("sniff parsing") do |check_exact|
+    it "should execute a sniff without error" do
+      expect { subject.check_sniff }.not_to raise_error
+    end
+
+    it "should return the correct sniff URL list" do
+      uris = subject.check_sniff
+      
+      # ES 1.x returned the public hostname by default. This is hard to approximate
+      # so for ES1.x we don't check the *exact* hostname
+      if check_exact
+        expect(uris).to include(::LogStash::Util::SafeURI.new("//#{get_host_port}"))
+      else
+        expect(uris.size).to eq(1)
+      end
+    end
+  end
+  
+  describe("Simple sniff parsing")  do
+    include_examples("sniff parsing", false)
+  end
+  
+  # We do a more thorough check on these versions because we can more reliably guess the ip
+  describe("Complex sniff parsing ES 5x/2x", :version_greater_than_equal_to_2x => true) do
+    include_examples("sniff parsing", true)
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -43,31 +43,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         expect(subject.sniffer_alive?).to eql(true)
       end
     end
-
-    describe "check sniff" do
-      context "with a good sniff result" do
-        let(:sniff_resp_path) { File.dirname(__FILE__) + '/../../../../fixtures/5x_node_resp.json' }
-        let(:sniff_resp) { double("resp") }
-        let(:sniff_resp_body) { File.open(sniff_resp_path).read }
-        
-        before do
-          allow(subject).to receive(:perform_request).
-                              with(:get, '_nodes').
-                              and_return([double('url'), sniff_resp])
-          allow(sniff_resp).to receive(:body).and_return(sniff_resp_body)
-        end
-        
-        it "should execute a sniff without error" do
-          expect { subject.check_sniff }.not_to raise_error
-        end
-
-        it "should return the correct sniff URL list" do
-          url_strs = subject.check_sniff.map(&:to_s)
-          expect(url_strs).to include("http://127.0.0.1:9200")
-          expect(url_strs).to include("http://127.0.0.1:9201")
-        end
-      end
-    end
   end
 
   describe "closing" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.4.0
+  version: 5.4.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-24 00:00:00.000000000 Z
+date: 2016-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -193,7 +193,6 @@ files:
 - lib/logstash/outputs/elasticsearch/template_manager.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
-- spec/fixtures/5x_node_resp.json
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
@@ -204,6 +203,7 @@ files:
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/secure_spec.rb
+- spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb
@@ -245,7 +245,6 @@ specification_version: 4
 summary: Logstash Output to Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
-- spec/fixtures/5x_node_resp.json
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
@@ -256,6 +255,7 @@ test_files:
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/secure_spec.rb
+- spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_5x_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/update_spec.rb

data/spec/fixtures/5x_node_resp.json

@@ -1,2 +0,0 @@
-
-{"cluster_name":"elasticsearch","nodes":{"vW8V2o_KSnOa-i97FACWFw":{"name":"Blockbuster","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9200","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Blockbuster"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19048,"mlockall":false},"jvm":{"pid":19048,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781724873,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9300","[::1]:9300","127.0.0.1:9300"],"publish_address":"127.0.0.1:9300","profiles":{}},"http":{"bound_address":["[fe80::1]:9200","[::1]:9200","127.0.0.1:9200"],"publish_address":"127.0.0.1:9200","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}},"BIAIepXSTYufETY06CUpYw":{"name":"Spyne","transport_address":"127.0.0.1:9301","host":"127.0.0.1","ip":"127.0.0.1","version":"5.0.0-alpha2","build_hash":"e3126df","http_address":"127.0.0.1:9201","roles":["master","data","ingest"],"settings":{"client":{"type":"node"},"cluster":{"name":"elasticsearch"},"node":{"name":"Spyne"},"path":{"logs":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2/logs","home":"/Users/andrewvc/Downloads/elasticsearch-5.0.0-alpha2"},"config":{"ignore_system_properties":"true"}},"os":{"refresh_interval_in_millis":1000,"name":"Mac OS X","arch":"x86_64","version":"10.11.4","available_processors":8,"allocated_processors":8},"process":{"refresh_interval_in_millis":1000,"id":19029,"mlockall":false},"jvm":{"pid":19029,"version":"1.8.0_51","vm_name":"Java HotSpot(TM) 64-Bit Server VM","vm_version":"25.51-b03","vm_vendor":"Oracle Corporation","start_time_in_millis":1463781692985,"mem":{"heap_init_in_bytes":268435456,"heap_max_in_bytes":1037959168,"non_heap_init_in_bytes":2555904,"non_heap_max_in_bytes":0,"direct_max_in_bytes":1037959168},"gc_collectors":["ParNew","ConcurrentMarkSweep"],"memory_pools":["Code Cache","Metaspace","Compressed Class Space","Par Eden Space","Par Survivor Space","CMS Old Gen"],"using_compressed_ordinary_object_pointers":"true"},"thread_pool":{"force_merge":{"type":"fixed","min":1,"max":1,"queue_size":-1},"fetch_shard_started":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"listener":{"type":"fixed","min":4,"max":4,"queue_size":-1},"index":{"type":"fixed","min":8,"max":8,"queue_size":200},"refresh":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"generic":{"type":"cached","keep_alive":"30s","queue_size":-1},"warmer":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"search":{"type":"fixed","min":13,"max":13,"queue_size":1000},"flush":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1},"fetch_shard_store":{"type":"scaling","min":1,"max":16,"keep_alive":"5m","queue_size":-1},"management":{"type":"scaling","min":1,"max":5,"keep_alive":"5m","queue_size":-1},"get":{"type":"fixed","min":8,"max":8,"queue_size":1000},"bulk":{"type":"fixed","min":8,"max":8,"queue_size":50},"snapshot":{"type":"scaling","min":1,"max":4,"keep_alive":"5m","queue_size":-1}},"transport":{"bound_address":["[fe80::1]:9301","[::1]:9301","127.0.0.1:9301"],"publish_address":"127.0.0.1:9301","profiles":{}},"http":{"bound_address":["[fe80::1]:9201","[::1]:9201","127.0.0.1:9201"],"publish_address":"127.0.0.1:9201","max_content_length_in_bytes":104857600},"plugins":[],"modules":[{"name":"ingest-grok","version":"5.0.0-alpha2","description":"Ingest processor that uses grok patterns to split text","classname":"org.elasticsearch.ingest.grok.IngestGrokPlugin"},{"name":"lang-expression","version":"5.0.0-alpha2","description":"Lucene expressions integration for Elasticsearch","classname":"org.elasticsearch.script.expression.ExpressionPlugin"},{"name":"lang-groovy","version":"5.0.0-alpha2","description":"Groovy scripting integration for Elasticsearch","classname":"org.elasticsearch.script.groovy.GroovyPlugin"},{"name":"lang-mustache","version":"5.0.0-alpha2","description":"Mustache scripting integration for Elasticsearch","classname":"org.elasticsearch.script.mustache.MustachePlugin"},{"name":"lang-painless","version":"5.0.0-alpha2","description":"An easy, safe and fast scripting language for Elasticsearch","classname":"org.elasticsearch.painless.PainlessPlugin"},{"name":"reindex","version":"5.0.0-alpha2","description":"The Reindex module adds APIs to reindex from one index to another or update documents in place.","classname":"org.elasticsearch.index.reindex.ReindexPlugin"}],"ingest":{"processors":[{"type":"append"},{"type":"convert"},{"type":"date"},{"type":"fail"},{"type":"foreach"},{"type":"grok"},{"type":"gsub"},{"type":"join"},{"type":"lowercase"},{"type":"remove"},{"type":"rename"},{"type":"set"},{"type":"split"},{"type":"trim"},{"type":"uppercase"}]}}}}