logstash-output-elasticsearch 6.0.0-java → 6.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c739e5dde0b8370dcc47e86b0dc2f92efb6bc6e
-  data.tar.gz: efe8aebc49fb32623f0c8ed65f6427325357da1d
+  metadata.gz: f8df743661ac7aed2fb52dddaf677ce1705a0612
+  data.tar.gz: 10bcb6b88076eafe00aea65c6d9e01d4d3fd8648
 SHA512:
-  metadata.gz: 3cdc83c314948cdc3ddf5323755a4e4ff44ffc9df004ca5deaa66fa34e1f6c86532cb368c426f68d27f910963563e0dcceef6baa625985e60259e5a681180f25
-  data.tar.gz: dc6950d258a89c4b5dbc96150527de923606b1011e248f6ce145c703fc817614329cc964f21375a1f6ad69a46e35c463366209d58350d7257006065d29d7239b
+  metadata.gz: 605c1958b1b4d22e51132e71932174c86c3d1696b80a7f35e07ec147301072dec9befeb64fae5bd240cff7c38f791fa3ede8bfa8a4781cd705a956d4a6e5b6b4
+  data.tar.gz: 69fae6a096cfa00b41037800d7a09737d0fec46b9722796f746d2aa1074be49147b3d2e8f2ec9ca39fdabf35130a2d603f8f512f771bceb09b1afdc1ffa6d0e8

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.1.0
+- Add option to use an absolute healthcheck path
+
 ## 6.0.0
 - Proxies requiring auth now always work when a URL is specified
 - It is no longer possible to specify a proxy as a hash due to security reasons 

data/lib/logstash/outputs/elasticsearch.rb

@@ -176,7 +176,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # to service requests. If you have custom firewall rules you may need to change
   # this
   config :healthcheck_path, :validate => :string, :default => "/"
-
+  # When a `healthcheck_path` config is provided, this additional flag can be used to 
+  # specify whether it is an absolute URI or a relative path. 
+  # If this flag is true, `healthcheck_path` is expected to be a fully formed URL 
+  # with any basic auth credentials provided in the URL itself.
+  config :absolute_healthcheck_path, :validate => :boolean, :default => false
   # How frequently, in seconds, to wait between resurrection attempts.
   # Resurrection is the process by which backend endpoints marked 'down' are checked
   # to see if they have come back to life

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -250,6 +250,7 @@ module LogStash; module Outputs; class ElasticSearch;
         :sniffing => sniffing,
         :sniffer_delay => options[:sniffer_delay],
         :healthcheck_path => options[:healthcheck_path],
+        :absolute_healthcheck_path => options[:absolute_healthcheck_path],
         :resurrect_delay => options[:resurrect_delay],
         :url_normalizer => self.method(:host_to_url)
       }

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -46,26 +46,13 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def perform_request(url, method, path, params={}, body=nil)
       params = (params || {}).merge @request_options
       params[:body] = body if body
-      
-      request_uri = if path
-                      # Combine the paths using the minimal # of /s
-                      # First, we make sure the path is relative so URI.join does
-                      # the right thing
-                      relative_path = path && path.start_with?("/") ? path[1..-1] : path
-                      # Wrap this with a safe URI defensively against careless handling later
-                      ::LogStash::Util::SafeURI.new(URI.join(url.uri, relative_path))
-                    else
-                      ::LogStash::Util::SafeURI.new(url.uri.clone)
-                    end
-        
-      # We excise auth info from the URL in case manticore itself tries to stick
-      # sensitive data in a thrown exception or log data
-      if request_uri.user
-        params[:auth] = { :user => request_uri.user, :password => request_uri.password, :eager => true }
-        request_uri.user = nil
-        request_uri.password = nil
+
+      if url.user
+        params[:auth] = { :user => url.user, :password => url.password, :eager => true }
       end
 
+      request_uri = format_url(url, path)
+
       resp = @manticore.send(method.downcase, request_uri.to_s, params)
 
       # Manticore returns lazy responses by default
@@ -83,6 +70,33 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       resp
     end
 
+    def format_url(url, path)
+
+      request_uri = url.uri
+
+      if path
+        # Combine the paths using the minimal # of /s
+        # First, we make sure the path is relative so URI.join does
+        # the right thing
+        relative_path = path && path.start_with?("/") ? path[1..-1] : path
+        # because URI.join obliterates the query parameter, we need to save it
+        # and restore it after this operation
+        query = request_uri.query
+        request_uri = URI.join(request_uri, relative_path)
+        request_uri.query = query
+      else
+        request_uri = request_uri.clone
+      end
+
+      # We excise auth info from the URL in case manticore itself tries to stick
+      # sensitive data in a thrown exception or log data
+      request_uri.user = nil
+      request_uri.password = nil
+
+      # Wrap this with a safe URI defensively against careless handling later
+      ::LogStash::Util::SafeURI.new(request_uri)
+    end
+
     def close
       @manticore.close
     end

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -27,10 +27,13 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path
+    attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :absolute_healthcheck_path
+
+    ROOT_URI_PATH = '/'.freeze
 
     DEFAULT_OPTIONS = {
-      :healthcheck_path => '/'.freeze,
+      :healthcheck_path => ROOT_URI_PATH,
+      :absolute_healthcheck_path => false,
       :scheme => 'http',
       :resurrect_delay => 5,
       :sniffing => false,
@@ -46,6 +49,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @url_normalizer = options[:url_normalizer]
       DEFAULT_OPTIONS.merge(options).tap do |merged|
         @healthcheck_path = merged[:healthcheck_path]
+        @absolute_healthcheck_path = merged[:absolute_healthcheck_path]
         @resurrect_delay = merged[:resurrect_delay]
         @sniffing = merged[:sniffing]
         @sniffer_delay = merged[:sniffer_delay]
@@ -225,11 +229,17 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
         begin
+          path = healthcheck_path
+          healthcheck_url = url
+          if @absolute_healthcheck_path
+            healthcheck_url = ::LogStash::Util::SafeURI.new(healthcheck_path)
+            path = ROOT_URI_PATH
+          end
           logger.info("Running health check to see if an Elasticsearch connection is working",
-                      url: url, healthcheck_path: healthcheck_path)
-          response = perform_request_to_url(url, :head, healthcheck_path)
+                        :healthcheck_url => healthcheck_url, :path => path)
+          response = perform_request_to_url(healthcheck_url, :head, path)
           # If no exception was raised it must have succeeded!
-          logger.warn("Restored connection to ES instance", :url => url.sanitized)
+          logger.warn("Restored connection to ES instance", :url => healthcheck_url.sanitized)
           @state_mutex.synchronize { meta[:state] = :alive }
         rescue HostUnreachableError, BadResponseCodeError => e
           logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: url.sanitized, error_type: e.class, error: e.message)

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -12,7 +12,8 @@ module LogStash; module Outputs; class ElasticSearch;
       common_options = {
         :client_settings => client_settings,
         :resurrect_delay => params["resurrect_delay"],
-        :healthcheck_path => params["healthcheck_path"]
+        :healthcheck_path => params["healthcheck_path"],
+        :absolute_healthcheck_path => params["absolute_healthcheck_path"]
       }
 
       if params["sniffing"]

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '6.0.0'
+  s.version         = '6.1.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -39,6 +39,34 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
     end
   end
 
+  describe "format_url" do
+    let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
+    let(:path) { "_bulk" }
+    subject { described_class.new(double("logger"), {}) }
+
+    it "should add the path argument to the uri's path" do
+      expect(subject.format_url(url, path).path).to eq("/path/_bulk")
+    end
+
+    context "when uri contains query parameters" do
+      let(:query_params) { "query=value&key=value2" }
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
+
+      it "should retain query_params after format" do
+        expect(subject.format_url(url, path).query).to eq(query_params)
+      end
+    end
+
+    context "when uri contains credentials" do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://user:pass@localhost:9200") }
+
+      it "should remove credentials after format" do
+        expect(subject.format_url(url, path).user).to be_nil
+        expect(subject.format_url(url, path).password).to be_nil
+      end
+    end
+  end
+
   describe "integration specs", :integration => true do
     it "should perform correct tests without error" do
       resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -9,26 +9,25 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
-  
+
   let(:manticore_double) { double("manticore a") }
   before do
-    allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
-    
-    
+
     response_double = double("manticore response").as_null_object
     # Allow healtchecks
     allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
     allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
-    
+    allow(manticore_double).to receive(:close)
+
     allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
     subject.start
   end
-  
+
   after do
     subject.close
   end
-  
+
   describe "initialization" do
     it "should be successful" do
       expect { subject }.not_to raise_error
@@ -44,6 +43,26 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       expect(subject).to receive(:healthcheck!).once
       sleep(subject.resurrect_delay + 1)
     end
+
+    describe "absolute_healthcheck_path" do
+      let(:options) { super.merge(:absolute_healthcheck_path => true, :healthcheck_path => "http://abc:xyz@localhost:9200")}
+      let(:pool) { described_class.new(logger, adapter, initial_urls, options) }
+
+      before do
+        pool.start
+      end
+
+      after do
+        pool.close
+      end
+
+      context "when enabled" do
+        it "should use the healthcheck_path as URL to do a health check" do
+          expect(adapter).to receive(:perform_request).with(::LogStash::Util::SafeURI.new(subject.healthcheck_path), :head, "/", {}, nil)
+          pool.healthcheck!
+        end
+      end
+    end
   end
 
   describe "the sniffer" do
@@ -88,7 +107,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       expect(subject).to have_received(:in_use_connections).twice
     end
   end
-  
+
   describe "connection management" do
     context "with only one URL in the list" do
       it "should use the only URL in 'with_connection'" do
@@ -99,8 +118,11 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     context "with multiple URLs in the list" do
+      before :each do
+        allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+      end
       let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
-      
+
       it "should minimize the number of connections to a single URL" do
         connected_urls = []
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 6.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-20 00:00:00.000000000 Z
+date: 2016-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement