logstash-output-elasticsearch 5.3.5-java → 5.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5166d95227401f888a2f3ed3606b0c6e42ee30e
-  data.tar.gz: 5e17a2dd88624f58bc2e3ba78c7f6d53a42b8e94
+  metadata.gz: d4c13cc28976cdad32da8417651ef69ad5695333
+  data.tar.gz: 145160a4c63cd206a3d869f044b434a169fe5f43
 SHA512:
-  metadata.gz: 0d93a1c0a567986a601429c7d30acbb0471c4f8edab1e7a0cc67eacd1eb3e6e402e8ed6ea63332ed16598447ca40820439726e047aeaa71c69c24012d9bcfffb
-  data.tar.gz: 09551c3614f59c4eba620fe42ff21078f00c3544001585deaa2fee95d2792b75662b3e7bb31b9bedaa012458c53752623aaf2efc8a1974578e388abb6c9e65c9
+  metadata.gz: 0256fab32a89ddafee3dbe18023c4d1bc415d32d1aab8bd30e11bac473760bc6c5fa9beab01b058ebee9e14ec4163f9ef1882527e5a5205592add85b63007b25
+  data.tar.gz: 297813ad0a6c5103c04fc9cd5085829bb8982bb56be9c15526561fc361d5094f2dbecbde3d5c9dd93b5d4fc1513c1be51dd6ddf8d71c73c15026e802c2411d46

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 5.4.0
+- Perform healthcheck against hosts right after startup / sniffing
+
 ## 5.3.5
 - Docs: Remove mention of using the elasticsearch_java output plugin because it is no longer supported
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -102,6 +102,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # not also set this field. That will raise an error at startup
   config :path, :validate => :string
 
+  # Pass a set of key value pairs as the URL query string. This query string is added
+  # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
+  # urls that already have query strings, the one specified here will be appended.
+  config :parameters, :validate => :hash
+
   # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
   # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
   # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -79,8 +79,8 @@ module LogStash; module Outputs; class ElasticSearch
       mod.config :hosts, :validate => :array, :default => ["127.0.0.1"]
 
       # This plugin uses the bulk index API for improved indexing performance.
-      # This setting defines the maximum sized bulk request Logstash will make
-      # You you may want to increase this to be in line with your pipeline's batch size.
+      # This setting defines the maximum sized bulk request Logstash will make.
+      # You may want to increase this to be in line with your pipeline's batch size.
       # If you specify a number larger than the batch size of your pipeline it will have no effect,
       # save for the case where a filter increases the size of an inflight batch by outputting
       # events.

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -131,7 +131,7 @@ module LogStash; module Outputs; class ElasticSearch;
       timeout = options[:timeout] || 0
 
       host_ssl_opt = client_settings[:ssl].nil? ? nil : client_settings[:ssl][:enabled]
-      urls = hosts.map {|host| host_to_url(host, host_ssl_opt, client_settings[:path])}
+      urls = hosts.map {|host| host_to_url(host, host_ssl_opt, client_settings[:path], client_settings[:parameters])}
 
       adapter_options = {
         :socket_timeout => timeout,
@@ -186,7 +186,7 @@ module LogStash; module Outputs; class ElasticSearch;
     HOSTNAME_PORT_REGEX=/\A(?<hostname>([A-Za-z0-9\.\-]+)|\[[0-9A-Fa-f\:]+\])(:(?<port>\d+))?\Z/
     URL_REGEX=/\A#{URI::regexp(['http', 'https'])}\z/
     # Parse a configuration host to a normalized URL
-    def host_to_url(host, ssl=nil, path=nil)
+    def host_to_url(host, ssl=nil, path=nil, parameters=nil)
       explicit_scheme = case ssl
                         when true
                           "https"
@@ -233,6 +233,15 @@ module LogStash; module Outputs; class ElasticSearch;
         url.path = path  # The URI library cannot stringify if it holds a nil
       end
 
+      if parameters
+        query_string = parameters.map { |k,v| "#{k}=#{v}" }.join("&")
+        if query = url.query
+          url.query = "#{query}&#{query_string}"
+        else
+          url.query = query_string
+        end
+      end
+
       if url.password || url.user
         raise LogStash::ConfigurationError, "We do not support setting the user password in the URL directly as " +
           "this may be logged to disk thus leaking credentials. Use the 'user' and 'password' options respectively"

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -31,7 +31,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def perform_request(url, method, path, params={}, body=nil)
       params = (params || {}).merge @request_options
       params[:body] = body if body
-      url_and_path = (url + path).to_s # Convert URI object to string
+      # Convert URI object to string
+      url_and_path = path ? (url + path).to_s  : url.to_s
 
       resp = @manticore.send(method.downcase, url_and_path, params)
 

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -62,8 +62,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # Holds metadata about all URLs
       @url_info = {}
       @stopping = false
-
+      
       update_urls(initial_urls)
+      
       start_resurrectionist
       start_sniffer if @sniffing
     end
@@ -96,7 +97,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     def alive_urls_count
-      @state_mutex.synchronize { @url_info.values.select {|v| !v[:dead] }.count }
+      @state_mutex.synchronize { @url_info.values.select {|v| !v[:state] == :alive }.count }
     end
 
     def url_info
@@ -186,24 +187,24 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def start_resurrectionist
       @resurrectionist = Thread.new do
         until_stopped("resurrection", @resurrect_delay) do
-          resurrect_dead!
+          healthcheck!
         end
       end
     end
 
-    def resurrect_dead!
+    def healthcheck!
       # Try to keep locking granularity low such that we don't affect IO...
-      @state_mutex.synchronize { @url_info.select {|url,meta| meta[:dead] } }.each do |url,meta|
+      @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
         safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
         begin
           logger.info("Running health check to see if an Elasticsearch connection is working",
                       url: safe_url, healthcheck_path: @healthcheck_path)
-          perform_request_to_url(url, "HEAD", @healthcheck_path)
+          response = perform_request_to_url(url, "HEAD", @healthcheck_path)
           # If no exception was raised it must have succeeded!
           logger.warn("Restored connection to ES instance", :url => safe_url)
-          @state_mutex.synchronize { meta[:dead] = false }
-        rescue HostUnreachableError => e
-          logger.debug("Attempted to resurrect connection to dead ES instance, but got an error.", url: safe_url, error_type: e.class, error: e.message)
+          @state_mutex.synchronize { meta[:state] = :alive }
+        rescue HostUnreachableError, BadResponseCodeError => e
+          logger.warn("Attempted to resurrect connection to dead ES instance, but got an error.", url: safe_url, error_type: e.class, error: e.message)
         end
       end
     end
@@ -280,6 +281,12 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
           logger.info("Elasticsearch pool URLs updated", :changes => safe_state_changes(state_changes))
         end
       end
+      
+      # Run an inline healthcheck anytime URLs are updated
+      # This guarantees that during startup / post-startup
+      # sniffing we don't have idle periods waiting for the
+      # periodic sniffer to allow new hosts to come online
+      healthcheck! 
     end
     
     def safe_state_changes(state_changes)
@@ -305,7 +312,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def empty_url_meta
       {
         :in_use => 0,
-        :dead => false
+        :state => :unknown
       }
     end
 
@@ -340,7 +347,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
         logger.warn("Marking url as dead.", :reason => error.message, :url => safe_url,
                     :error_message => error.message, :error_class => error.class.name)
-        meta[:dead] = true
+        meta[:state] = :dead
         meta[:last_error] = error
         meta[:last_errored_at] = Time.now
       end
@@ -361,7 +368,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         lowest_value_seen = nil
         @url_info.each do |url,meta|
           meta_in_use = meta[:in_use]
-          next if meta[:dead]
+          next if meta[:state] == :dead
 
           if lowest_value_seen.nil? || meta_in_use < lowest_value_seen
             lowest_value_seen = meta_in_use

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -24,6 +24,10 @@ module LogStash; module Outputs; class ElasticSearch;
         client_settings[:path] = "/#{params["path"]}/".gsub(/\/+/, "/") # Normalize slashes
       end
 
+      if params["parameters"]
+        client_settings[:parameters] = params["parameters"]
+      end
+
       logger.debug? && logger.debug("Normalizing http path", :path => params["path"], :normalized => client_settings[:path])
 
       client_settings.merge! setup_ssl(logger, params)

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '5.3.5'
+  s.version         = '5.4.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -9,6 +9,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
   let(:options) { {:resurrect_delay => 2} } # Shorten the delay a bit to speed up tests
 
   subject { described_class.new(logger, adapter, initial_urls, options) }
+  
+  before do
+    allow(adapter).to receive(:perform_request).with(anything, 'HEAD', subject.healthcheck_path, {}, nil)
+  end
 
   describe "initialization" do
     it "should be successful" do
@@ -22,7 +26,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     it "should attempt to resurrect connections after the ressurrect delay" do
-      expect(subject).to receive(:resurrect_dead!).once
+      expect(subject).to receive(:healthcheck!).once
       sleep(subject.resurrect_delay + 1)
     end
   end
@@ -148,9 +152,9 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
         subject.return_connection(u)
         subject.mark_dead(u, Exception.new)
 
-        expect(subject.url_meta(u)[:dead]).to eql(true)
+        expect(subject.url_meta(u)[:state]).to eql(:dead)
         sleep subject.resurrect_delay + 1
-        expect(subject.url_meta(u)[:dead]).to eql(false)
+        expect(subject.url_meta(u)[:state]).to eql(:alive)
       end
     end
   end

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -1,6 +1,7 @@
 require_relative "../../../spec/es_spec_helper"
 require 'stud/temporary'
 require "logstash/outputs/elasticsearch"
+require 'manticore/client'
 
 describe "Proxy option" do
   let(:settings) {
@@ -14,7 +15,7 @@ describe "Proxy option" do
   }
 
   before do
-    allow(::Manticore::Client).to receive(:new).with(any_args)
+    allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
   end
 
   describe "valid configs" do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -128,6 +128,7 @@ describe "outputs/elasticsearch" do
         end
       end
     end
+
     describe "without a port specified" do
       it "should properly set the default port (9200) on the HTTP client" do
         expect(manticore_url.port).to eql(9200)
@@ -285,7 +286,7 @@ describe "outputs/elasticsearch" do
     subject(:eso) { LogStash::Outputs::ElasticSearch.new("validate_after_inactivity" => validate_after_inactivity) }
 
     before do
-      allow(::Manticore::Client).to receive(:new).with(any_args)
+      allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
       subject.register
     end
 
@@ -295,4 +296,91 @@ describe "outputs/elasticsearch" do
       end
     end
   end
+
+  describe "custom parameters" do
+
+    let(:eso) {LogStash::Outputs::ElasticSearch.new(options)}
+
+    let(:manticore_urls) { eso.client.pool.urls }
+    let(:manticore_url) { manticore_urls.first }
+
+    let(:custom_parameters_hash) { { "id" => 1, "name" => "logstash" } }
+    let(:custom_parameters_query) { custom_parameters_hash.map {|k,v| "#{k}=#{v}" }.join("&") }
+
+    before(:each) do
+      eso.register
+    end
+
+    after(:each) do
+      eso.close rescue nil
+    end
+
+    context "using non-url hosts" do
+
+      let(:options) {
+        {
+          "index" => "my-index",
+          "hosts" => ["localhost:9202"],
+          "path" => "some-path",
+          "parameters" => custom_parameters_hash
+        }
+      }
+
+      it "creates a URI with the added parameters" do
+        expect(eso.parameters).to eql(custom_parameters_hash)
+      end
+
+      it "sets the query string on the HTTP client" do
+        expect(manticore_url.query).to eql(custom_parameters_query)
+      end
+    end
+
+    context "using url hosts" do
+
+      context "with embedded query parameters" do
+        let(:options) {
+          { "hosts" => ["http://localhost:9202/path?#{custom_parameters_query}"] }
+        }
+
+        it "sets the query string on the HTTP client" do
+          expect(manticore_url.query).to eql(custom_parameters_query)
+        end
+      end
+
+      context "with explicity query parameters" do
+        let(:options) {
+          {
+            "hosts" => ["http://localhost:9202/path"],
+            "parameters" => custom_parameters_hash
+          }
+        }
+
+        it "sets the query string on the HTTP client" do
+          expect(manticore_url.query).to eql(custom_parameters_query)
+        end
+      end
+
+      context "with explicity query parameters and existing url parameters" do
+        let(:existing_query_string) { "existing=param" }
+        let(:options) {
+          {
+            "hosts" => ["http://localhost:9202/path?#{existing_query_string}"],
+            "parameters" => custom_parameters_hash
+          }
+        }
+
+        it "keeps the existing query string" do
+          expect(manticore_url.query).to include(existing_query_string)
+        end
+
+        it "includes the new query string" do
+          expect(manticore_url.query).to include(custom_parameters_query)
+        end
+
+        it "appends the new query string to the existing one" do
+          expect(manticore_url.query).to eql("#{existing_query_string}&#{custom_parameters_query}")
+        end
+      end
+    end
+  end
 end

data/spec/unit/outputs/elasticsearch_ssl_spec.rb

@@ -1,5 +1,6 @@
 require_relative "../../../spec/es_spec_helper"
 require 'stud/temporary'
+require "logstash/outputs/elasticsearch"
 
 describe "SSL option" do
   context "when using ssl without cert verification" do
@@ -16,7 +17,7 @@ describe "SSL option" do
     end
 
     it "should pass the flag to the ES client" do
-      expect(::Manticore::Client).to receive(:new) do |args|
+      expect(::Manticore::Client).to receive(:new).and_call_original do |args|
         expect(args[:ssl]).to eq(:enabled => true, :verify => false)
       end
       subject.register
@@ -32,6 +33,9 @@ describe "SSL option" do
 
   context "when using ssl with client certificates" do
     let(:keystore_path) { Stud::Temporary.file.path }
+    before do
+      `openssl req -x509  -batch -nodes -newkey rsa:2048 -keyout lumberjack.key -out #{keystore_path}.pem`
+    end
 
     after :each do
       File.delete(keystore_path)
@@ -42,8 +46,7 @@ describe "SSL option" do
       settings = {
         "hosts" => "node01",
         "ssl" => true,
-        "keystore" => keystore_path,
-        "keystore_password" => "test"
+        "cacert" => keystore_path,
       }
       next LogStash::Outputs::ElasticSearch.new(settings)
     end
@@ -51,7 +54,7 @@ describe "SSL option" do
     it "should pass the keystore parameters to the ES client" do
       expect(::Manticore::Client).to receive(:new) do |args|
         expect(args[:ssl]).to include(:keystore => keystore_path, :keystore_password => "test")
-      end
+      end.and_call_original
       subject.register
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.3.5
+  version: 5.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-22 00:00:00.000000000 Z
+date: 2016-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement