logstash-output-elasticsearch 5.4.1-java → 6.0.0-java

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -6,7 +6,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       def initialize(response_code, url, body)
         @response_code = response_code
-        @url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
+        @url = url
         @body = body
       end
 
@@ -19,7 +19,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       def initialize(original_error, url)
         @original_error = original_error
-        @url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
+        @url = url
       end
 
       def message
@@ -27,13 +27,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :auth, :healthcheck_path
+    attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path
 
     DEFAULT_OPTIONS = {
       :healthcheck_path => '/'.freeze,
       :scheme => 'http',
       :resurrect_delay => 5,
-      :auth => nil, # Can be set to {:user => 'user', :password => 'pass'}
       :sniffing => false,
       :sniffer_delay => 10,
     }.freeze
@@ -41,30 +40,27 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def initialize(logger, adapter, initial_urls=[], options={})
       @logger = logger
       @adapter = adapter
-
+      @initial_urls = initial_urls
+      
+      raise ArgumentError, "No URL Normalizer specified!" unless options[:url_normalizer]
+      @url_normalizer = options[:url_normalizer]
       DEFAULT_OPTIONS.merge(options).tap do |merged|
         @healthcheck_path = merged[:healthcheck_path]
-        @scheme = merged[:scheme]
         @resurrect_delay = merged[:resurrect_delay]
-        @auth = merged[:auth]
         @sniffing = merged[:sniffing]
         @sniffer_delay = merged[:sniffer_delay]
       end
 
-      # Override the scheme if one is explicitly set in urls
-      if initial_urls.any? {|u| u.scheme == 'https'} && @scheme == 'http'
-        raise ArgumentError, "HTTP was set as scheme, but an HTTPS URL was passed in!"
-      end
-
       # Used for all concurrent operations in this class
       @state_mutex = Mutex.new
 
       # Holds metadata about all URLs
       @url_info = {}
       @stopping = false
-      
-      update_urls(initial_urls)
-      
+    end
+    
+    def start
+      update_urls(@initial_urls)
       start_resurrectionist
       start_sniffer if @sniffing
     end
@@ -204,7 +200,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         if matches
           host = matches[1].empty? ? matches[2] : matches[1]
           port = matches[3]
-          LogStash::Util::SafeURI.new("#{host}:#{port}")
+          ::LogStash::Util::SafeURI.new("#{host}:#{port}")
         end
       end.compact
     end
@@ -228,26 +224,25 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def healthcheck!
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
-        safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
         begin
           logger.info("Running health check to see if an Elasticsearch connection is working",
-                      url: safe_url, healthcheck_path: @healthcheck_path)
-          response = perform_request_to_url(url, "HEAD", @healthcheck_path)
+                      url: url, healthcheck_path: healthcheck_path)
+          response = perform_request_to_url(url, :head, healthcheck_path)
           # If no exception was raised it must have succeeded!
-          logger.warn("Restored connection to ES instance", :url => safe_url)
+          logger.warn("Restored connection to ES instance", :url => url.sanitized)
           @state_mutex.synchronize { meta[:state] = :alive }
         rescue HostUnreachableError, BadResponseCodeError => e
-          logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: safe_url, error_type: e.class, error: e.message)
+          logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: url.sanitized, error_type: e.class, error: e.message)
         end
       end
     end
 
     def stop_resurrectionist
-      @resurrectionist.join
+      @resurrectionist.join if @resurrectionist
     end
 
     def resurrectionist_alive?
-      @resurrectionist.alive?
+      @resurrectionist ? @resurrectionist.alive? : nil
     end
 
     def perform_request(method, path, params={}, body=nil)
@@ -270,18 +265,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def normalize_url(uri)
-      raise ArgumentError, "Only URI/SafeURI objects may be passed in!" unless uri.is_a?(URI) || uri.is_a?(LogStash::Util::SafeURI)
-      uri = uri.clone
-
-      # Set credentials if need be
-      if @auth && !uri.user
-        uri.user ||= @auth[:user]
-        uri.password ||= @auth[:password]
+      u = @url_normalizer.call(uri)
+      if !u.is_a?(::LogStash::Util::SafeURI)
+        raise "URL Normalizer returned a '#{u.class}' rather than a SafeURI! This shouldn't happen!"
       end
-
-      uri.scheme = @scheme
-
-      uri
+      u
     end
 
     def update_urls(new_urls)
@@ -313,7 +301,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       if state_changes[:removed].size > 0 || state_changes[:added].size > 0
         if logger.info?
-          logger.info("Elasticsearch pool URLs updated", :changes => safe_state_changes(state_changes))
+          logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
         end
       end
       
@@ -324,14 +312,6 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       healthcheck! 
     end
     
-    def safe_state_changes(state_changes)
-      state_changes.reduce({}) do |acc, kv|
-        k,v = kv
-        acc[k] = v.map(&LogStash::Outputs::ElasticSearch::SafeURL.method(:without_credentials)).map(&:to_s)
-        acc
-      end
-    end
-
     def size
       @state_mutex.synchronize { @url_info.size }
     end
@@ -378,10 +358,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         meta = @url_info[url]
         # In case a sniff happened removing the metadata just before there's nothing to mark
         # This is an extreme edge case, but it can happen!
-        return unless meta 
-        safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
-        logger.warn("Marking url as dead.", :reason => error.message, :url => safe_url,
-                    :error_message => error.message, :error_class => error.class.name)
+        return unless meta
+        logger.warn("Marking url as dead. Last error: [#{error.class}] #{error.message}",
+                    :url => url, :error_message => error.message, :error_class => error.class.name)
         meta[:state] = :dead
         meta[:last_error] = error
         meta[:last_errored_at] = Time.now

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -6,7 +6,9 @@ module LogStash; module Outputs; class ElasticSearch;
         :pool_max_per_route => params["pool_max_per_route"],
         :check_connection_timeout => params["validate_after_inactivity"]
       }
-
+      
+      client_settings[:proxy] = params["proxy"] if params["proxy"]
+      
       common_options = {
         :client_settings => client_settings,
         :resurrect_delay => params["resurrect_delay"],
@@ -31,7 +33,6 @@ module LogStash; module Outputs; class ElasticSearch;
       logger.debug? && logger.debug("Normalizing http path", :path => params["path"], :normalized => client_settings[:path])
 
       client_settings.merge! setup_ssl(logger, params)
-      client_settings.merge! setup_proxy(logger, params)
       common_options.merge! setup_basic_auth(logger, params)
 
       # Update API setup
@@ -59,25 +60,8 @@ module LogStash; module Outputs; class ElasticSearch;
       )
     end
 
-    def self.setup_proxy(logger, params)
-      proxy = params["proxy"]
-      return {} unless proxy
-
-      # Symbolize keys
-      proxy = if proxy.is_a?(Hash)
-                Hash[proxy.map {|k,v| [k.to_sym, v]}]
-              elsif proxy.is_a?(String)
-                proxy
-              else
-                raise LogStash::ConfigurationError, "Expected 'proxy' to be a string or hash, not '#{proxy}''!"
-              end
-
-      return {:proxy => proxy}
-    end
-
     def self.setup_ssl(logger, params)
-      # If we have HTTPS hosts we act like SSL is enabled
-      params["ssl"] = true if params["hosts"].any? {|h| h.start_with?("https://")}
+      params["ssl"] = true if params["hosts"].any? {|h| h.scheme == "https" }
       return {} if params["ssl"].nil?
 
       return {:ssl => {:enabled => false}} if params["ssl"] == false

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -8,7 +8,7 @@ module LogStash; module Outputs; class ElasticSearch
       plugin.logger.info("Attempting to install template", :manage_template => template)
       install(plugin.client, plugin.template_name, template, plugin.template_overwrite)
     rescue => e
-      plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name)
+      plugin.logger.error("Failed to install template.", :message => e.message, :class => e.class.name, :backtrace => e.backtrace)
     end
 
     private

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '5.4.1'
+  s.version         = '6.0.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/sniffer_spec.rb

@@ -7,10 +7,14 @@ describe "pool sniffer", :integration => true do
   let(:logger) { Cabin::Channel.get }
   let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
-  let(:options) { {:resurrect_delay => 2} } # Shorten the delay a bit to speed up tests
+  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
 
   subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
   
+  before do
+    subject.start
+  end
+  
   shared_examples("sniff parsing") do |check_exact|
     it "should execute a sniff without error" do
       expect { subject.check_sniff }.not_to raise_error

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -9,16 +9,39 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
 
   it "should raise an exception if requests are issued after close" do
     subject.close
-    expect { subject.perform_request("http://localhost:9200", :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
+    expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
   end
 
   it "should implement host unreachable exceptions" do
     expect(subject.host_unreachable_exceptions).to be_a(Array)
   end
+  
+  describe "auth" do
+    let(:user) { "myuser" }
+    let(:password) { "mypassword" }
+    let(:noauth_uri) { clone = uri.uri.clone;clone.user=nil; clone.password=nil; clone.to_s }
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
+    
+    it "should convert the auth to params" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(200)
+      expect(subject.manticore).to receive(:get).
+        with(noauth_uri, {
+          :auth => {
+            :user => user,
+            :password => password,
+            :eager => true
+          }
+        }).and_return resp
+      
+      subject.perform_request(uri, :get, "/")
+    end
+  end
 
   describe "integration specs", :integration => true do
     it "should perform correct tests without error" do
-      resp = subject.perform_request("http://localhost:9200", :get, "/")
+      resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
       expect(resp.code).to eql(200)
     end
   end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -5,15 +5,30 @@ require "json"
 describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
   let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger) }
-  let(:initial_urls) { [URI.parse("http://localhost:9200")] }
-  let(:options) { {:resurrect_delay => 2} } # Shorten the delay a bit to speed up tests
+  let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }
+  let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
   
+  let(:manticore_double) { double("manticore a") }
   before do
-    allow(adapter).to receive(:perform_request).with(anything, 'HEAD', subject.healthcheck_path, {}, nil)
-  end
+    allow(adapter).to receive(:perform_request).with(anything, :head, subject.healthcheck_path, {}, nil)
+    
+    
+    response_double = double("manticore response").as_null_object
+    # Allow healtchecks
+    allow(manticore_double).to receive(:head).with(any_args).and_return(response_double)
+    allow(manticore_double).to receive(:get).with(any_args).and_return(response_double)
+    
+    allow(::Manticore::Client).to receive(:new).and_return(manticore_double)
 
+    subject.start
+  end
+  
+  after do
+    subject.close
+  end
+  
   describe "initialization" do
     it "should be successful" do
       expect { subject }.not_to raise_error
@@ -51,6 +66,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       allow(adapter).to receive(:close).and_call_original
       allow(subject).to receive(:wait_for_in_use_connections).and_call_original
       allow(subject).to receive(:in_use_connections).and_return([subject.empty_url_meta()],[])
+      allow(subject).to receive(:start)
       subject.close
     end
 
@@ -73,33 +89,18 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
   end
   
-  describe "safe_state_changes" do
-    let(:state_changes) do 
-      {
-        :added => [URI.parse("http://sekretu:sekretp@foo1")],
-        :removed => [URI.parse("http://sekretu:sekretp@foo2")]
-      }
-    end
-    let(:processed) { subject.safe_state_changes(state_changes)}
-    
-    it "should hide passwords" do
-      expect(processed[:added].any? {|p| p =~ /sekretp/ }).to be false
-      expect(processed[:removed].any? {|p| p =~ /sekretp/ }).to be false
-    end
-  end
-
   describe "connection management" do
     context "with only one URL in the list" do
       it "should use the only URL in 'with_connection'" do
         subject.with_connection do |c|
-          expect(c).to eql(initial_urls.first)
+          expect(c).to eq(initial_urls.first)
         end
       end
     end
 
     context "with multiple URLs in the list" do
-      let(:initial_urls) { [ URI.parse("http://localhost:9200"), URI.parse("http://localhost:9201"), URI.parse("http://localhost:9202") ] }
-
+      let(:initial_urls) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200"), ::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202") ] }
+      
       it "should minimize the number of connections to a single URL" do
         connected_urls = []
 
@@ -122,7 +123,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
         # The resurrectionist will call this to check on the backend
         response = double("response")
-        expect(adapter).to receive(:perform_request).with(u, 'HEAD', subject.healthcheck_path, {}, nil).and_return(response)
+        expect(adapter).to receive(:perform_request).with(u, :head, subject.healthcheck_path, {}, nil).and_return(response)
 
         subject.return_connection(u)
         subject.mark_dead(u, Exception.new)

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -3,7 +3,19 @@ require "logstash/outputs/elasticsearch/http_client"
 require "java"
 
 describe LogStash::Outputs::ElasticSearch::HttpClient do
-  let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get} }
+  let(:ssl) { nil }
+  let(:base_options) do
+    opts = {
+      :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
+      :logger => Cabin::Channel.get
+    }
+
+    if !ssl.nil? # Shortcut to set this
+      opts[:client_settings] = {:ssl => {:enabled => ssl}}
+    end
+
+    opts
+  end
 
   describe "Host/URL Parsing" do
     subject { described_class.new(base_options) }
@@ -12,71 +24,83 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     let(:ipv6_hostname) { "[::1]" }
     let(:ipv4_hostname) { "127.0.0.1" }
     let(:port) { 9202 }
-    let(:hostname_port) { "#{hostname}:#{port}"}
-    let(:http_hostname_port) { "http://#{hostname_port}"}
-    let(:https_hostname_port) { "https://#{hostname_port}"}
-    let(:http_hostname_port_path) { "http://#{hostname_port}/path"}
-
+    let(:hostname_port) { "#{hostname}:#{port}" }
+    let(:hostname_port_uri) { ::LogStash::Util::SafeURI.new("//#{hostname_port}") }
+    let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") }
+    let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") }
+    let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") }
+    
     shared_examples("proper host handling") do
       it "should properly transform a host:port string to a URL" do
-        expect(subject.send(:host_to_url, hostname_port).to_s).to eql(http_hostname_port)
-      end
-
-      it "should raise an error when a partial URL is an invalid format" do
-        expect {
-          subject.send(:host_to_url, "#{hostname_port}/")
-        }.to raise_error(LogStash::ConfigurationError)
+        expect(subject.host_to_url(hostname_port_uri)).to eq(http_hostname_port)
       end
 
       it "should not raise an error with a / for a path" do
-        expect(subject.send(:host_to_url, "#{http_hostname_port}/").to_s).to eql("#{http_hostname_port}/")
+        expect(subject.host_to_url(::LogStash::Util::SafeURI.new("#{http_hostname_port}/"))).to eq(LogStash::Util::SafeURI.new("#{http_hostname_port}/"))
       end
 
       it "should parse full URLs correctly" do
-        expect(subject.send(:host_to_url, http_hostname_port).to_s).to eql(http_hostname_port)
-      end
-
-      it "should reject full URLs with usernames and passwords" do
-        expect {
-          subject.send(:host_to_url, "http://user:password@host.domain")
-        }.to raise_error(LogStash::ConfigurationError)
+        expect(subject.host_to_url(http_hostname_port)).to eq(http_hostname_port)
       end
 
       describe "ssl" do
-        it "should refuse to handle an http url when ssl is true" do
-          expect {
-            subject.send(:host_to_url, http_hostname_port, true)
-          }.to raise_error(LogStash::ConfigurationError)
+        context "when SSL is true" do
+          let(:ssl) { true }
+          let(:base_options) { super.merge(:hosts => [http_hostname_port]) }
+
+          it "should refuse to handle an http url" do
+            expect {
+              subject.host_to_url(http_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
         end
 
-        it "should refuse to handle an https url when ssl is false" do
-          expect {
-            subject.send(:host_to_url, https_hostname_port, false)
-          }.to raise_error(LogStash::ConfigurationError)
+        context "when SSL is false" do
+          let(:ssl) { false }
+          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          
+          it "should refuse to handle an https url" do
+            expect {
+              subject.host_to_url(https_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
         end
 
-        it "should handle an ssl url correctly when SSL is nil" do
-          expect(subject.send(:host_to_url, https_hostname_port, nil).to_s).to eql(https_hostname_port)
-        end
-
-        it "should raise an exception if an unexpected value is passed in" do
-          expect { subject.send(:host_to_url, https_hostname_port, {})}.to raise_error(ArgumentError)
-        end
+        describe "ssl is nil" do
+          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          it "should handle an ssl url correctly when SSL is nil" do
+            subject
+            expect(subject.host_to_url(https_hostname_port)).to eq(https_hostname_port)
+          end
+        end       
       end
 
       describe "path" do
+        let(:url) { http_hostname_port_path }
+        let(:base_options) { super.merge(:hosts => [url]) }
+        
         it "should allow paths in a url" do
-          expect(subject.send(:host_to_url, http_hostname_port_path, nil).to_s).to eql(http_hostname_port_path)
+          expect(subject.host_to_url(url)).to eq(url)
         end
 
-        it "should not allow paths in two places" do
-          expect {
-            subject.send(:host_to_url, http_hostname_port_path, false, "/otherpath")
-          }.to raise_error(LogStash::ConfigurationError)
+        context "with the path option set" do
+          let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
+          
+          it "should not allow paths in two places" do
+            expect {
+              subject.host_to_url(url)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
         end
-
-        it "should automatically insert a / in front of path overlays if needed" do
-          expect(subject.send(:host_to_url, http_hostname_port, false, "otherpath")).to eql(URI.parse(http_hostname_port + "/otherpath"))
+        
+        context "with a path missing a leading /" do
+          let(:url) { http_hostname_port }
+          let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
+          
+          
+          it "should automatically insert a / in front of path overlays" do
+            expect(subject.host_to_url(url)).to eq(LogStash::Util::SafeURI.new(url + "/otherpath"))
+          end
         end
       end
     end
@@ -97,6 +121,49 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
+  describe "join_bulk_responses" do
+    subject { described_class.new(base_options) }
+
+    context "when items key is available" do
+      require "json"
+      let(:bulk_response) {
+        LogStash::Json.load ('[{
+          "items": [{
+            "delete": {
+              "_index":   "website",
+              "_type":    "blog",
+              "_id":      "123",
+              "_version": 2,
+              "status":   200,
+              "found":    true
+            }
+          }],
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 1
+      end
+    end
+
+    context "when items key is not available" do
+      require "json"
+      let(:bulk_response) {
+        JSON.parse ('[{
+          "took": 4,
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 0
+      end
+    end
+  end
+
   describe "sniffing" do
     let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }