logstash-output-elasticsearch 5.4.1-java → 6.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d76c66e378b2ccf47fd7cbf0358369eeddc13f9f
-  data.tar.gz: c91f5e96604d6eccaff69da784e234baec2d8f05
+  metadata.gz: 9c739e5dde0b8370dcc47e86b0dc2f92efb6bc6e
+  data.tar.gz: efe8aebc49fb32623f0c8ed65f6427325357da1d
 SHA512:
-  metadata.gz: 79370d08703d8cd92360d1487b5bcd4cdfde5927abc9f8b7525569d298059c7c10f0f165942d923e2ca7e63f88a3409ee1366ec3a60fa3b8adf261081f0f5e6e
-  data.tar.gz: 8bb54f67ecd81847ebc3a296f5249ed70add26b466016fde9cb869962ca0f28916b74cd76278317a2b7c45725cddd64bc4c0346094bda5d6d90a13a3c2f30bf4
+  metadata.gz: 3cdc83c314948cdc3ddf5323755a4e4ff44ffc9df004ca5deaa66fa34e1f6c86532cb368c426f68d27f910963563e0dcceef6baa625985e60259e5a681180f25
+  data.tar.gz: dc6950d258a89c4b5dbc96150527de923606b1011e248f6ce145c703fc817614329cc964f21375a1f6ad69a46e35c463366209d58350d7257006065d29d7239b

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 6.0.0
+- Proxies requiring auth now always work when a URL is specified
+- It is no longer possible to specify a proxy as a hash due to security reasons 
+- Fix URL normalization logic to correctly apply all settings to sniffed hosts
+- Proxies requiring auth now always work when a URL is specified
+- Switch internals to new LogStash::Util::SafeURI type for more defensive approach to logging credentials
+
 ## 5.4.1
 - Correctly sniff against ES 5.x clusters
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -8,6 +8,7 @@ require "stud/buffer"
 require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
+require "forwardable"
 
 # This plugin is the recommended method of storing logs in Elasticsearch.
 # If you plan on using the Kibana web interface, you'll want to use this output.
@@ -145,10 +146,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   config :sniffing_delay, :validate => :number, :default => 5
 
   # Set the address of a forward HTTP proxy.
-  # Can be either a string, such as `http://localhost:123` or a hash in the form
-  # of `{host: 'proxy.org' port: 80 scheme: 'http'}`.
-  # Note, this is NOT a SOCKS proxy, but a plain HTTP proxy
-  config :proxy
+  # This used to accept hashes as arguments but now only accepts
+  # arguments of the URI type to prevent leaking credentials.
+  config :proxy, :validate => :uri
 
   # Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
   # a timeout occurs, the request will be retried.
@@ -193,11 +193,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   config :validate_after_inactivity, :validate => :number, :default => 10000
 
   def build_client
-    @client = ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
+    @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
   end
 
   def close
     @stopping.make_true
+    @client.close if @client
   end
 
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -14,7 +14,7 @@ module LogStash; module Outputs; class ElasticSearch;
       install_template
       check_action_validity
 
-      @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts)
+      @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized))
     end
 
     # Receive an array of events and immediately attempt to index them (no buffering)
@@ -212,15 +212,14 @@ module LogStash; module Outputs; class ElasticSearch;
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         if RETRYABLE_CODES.include?(e.response_code)
-          safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(e.url)
-          log_hash = {:code => e.response_code, :url => safe_url}
+          log_hash = {:code => e.response_code, :url => e.url.sanitized}
           log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
           @logger.error("Attempted to send a bulk request to elasticsearch but received a bad HTTP response code!", log_hash)
 
           sleep_interval = sleep_for_interval(sleep_interval)
           retry unless @stopping.true?
         else
-          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code, :body => e.response.body)
+          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code, :body => e.body)
         end
       rescue => e
         # Stuff that should never happen

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -1,3 +1,5 @@
+require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978
+
 module LogStash; module Outputs; class ElasticSearch
   module CommonConfigs
     def self.included(mod)
@@ -76,7 +78,7 @@ module LogStash; module Outputs; class ElasticSearch
       #     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
       # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
       # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
-      mod.config :hosts, :validate => :array, :default => ["127.0.0.1"]
+      mod.config :hosts, :validate => :uri, :default => [::LogStash::Util::SafeURI.new("//127.0.0.1")], :list => true
 
       # This plugin uses the bulk index API for improved indexing performance.
       # This setting defines the maximum sized bulk request Logstash will make.

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -3,6 +3,7 @@ require "cabin"
 require "base64"
 require 'logstash/outputs/elasticsearch/http_client/pool'
 require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
+require 'cgi'
 
 module LogStash; module Outputs; class ElasticSearch;
   # This is a constant instead of a config option because
@@ -27,15 +28,48 @@ module LogStash; module Outputs; class ElasticSearch;
     #   :setting => value
     # }
 
+#
+    # The `options` is a hash where the following symbol keys have meaning:
+    #
+    # * `:hosts` - array of String. Set a list of hosts to use for communication.
+    # * `:port` - number. set the port to use to communicate with Elasticsearch
+    # * `:user` - String. The user to use for authentication.
+    # * `:password` - String. The password to use for authentication.
+    # * `:timeout` - Float. A duration value, in seconds, after which a socket
+    #    operation or request will be aborted if not yet successfull
+    # * `:client_settings` - a hash; see below for keys.
+    #
+    # The `client_settings` key is a has that can contain other settings:
+    #
+    # * `:ssl` - Boolean. Enable or disable SSL/TLS.
+    # * `:proxy` - String. Choose a HTTP HTTProxy to use.
+    # * `:path` - String. The leading path for prefixing Elasticsearch
+    #   requests. This is sometimes used if you are proxying Elasticsearch access
+    #   through a special http path, such as using mod_rewrite.
     def initialize(options={})
       @logger = options[:logger]
+      
       # Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
       # @options = DEFAULT_OPTIONS.merge(options)
       @options = options
+      
+      @url_template = build_url_template
+
       @pool = build_pool(@options)
       # mutex to prevent requests and sniffing to access the
       # connection pool at the same time
     end
+    
+    def build_url_template
+      {
+        :scheme => self.scheme,
+        :user => self.user,
+        :password => self.password,
+        :host => "URLTEMPLATE",
+        :port => self.port,
+        :path => self.path
+      }
+    end
 
     def template_install(name, template, force=false)
       if template_exists?(name) && !force
@@ -56,7 +90,6 @@ module LogStash; module Outputs; class ElasticSearch;
 
       return if actions.empty?
 
-
       bulk_actions = actions.collect do |action, args, source|
         args, source = update_action_builder(args, source) if action == 'update'
 
@@ -91,7 +124,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def join_bulk_responses(bulk_responses)
       {
         "errors" => bulk_responses.any? {|r| r["errors"] == true},
-        "items" => bulk_responses.reduce([]) {|m,r| m.concat(r["items"])}
+        "items" => bulk_responses.reduce([]) {|m,r| m.concat(r.fetch("items", []))}
       }
     end
 
@@ -105,34 +138,86 @@ module LogStash; module Outputs; class ElasticSearch;
       @pool.close
     end
 
-    private
+    
+    def calculate_property(uris, property, default, sniff_check)
+      values = uris.map(&property).uniq
 
-    #
-    # The `options` is a hash where the following symbol keys have meaning:
-    #
-    # * `:hosts` - array of String. Set a list of hosts to use for communication.
-    # * `:port` - number. set the port to use to communicate with Elasticsearch
-    # * `:user` - String. The user to use for authentication.
-    # * `:password` - String. The password to use for authentication.
-    # * `:timeout` - Float. A duration value, in seconds, after which a socket
-    #    operation or request will be aborted if not yet successfull
-    # * `:client_settings` - a hash; see below for keys.
-    #
-    # The `client_settings` key is a has that can contain other settings:
-    #
-    # * `:ssl` - Boolean. Enable or disable SSL/TLS.
-    # * `:proxy` - String. Choose a HTTP HTTProxy to use.
-    # * `:path` - String. The leading path for prefixing Elasticsearch
-    #   requests. This is sometimes used if you are proxying Elasticsearch access
-    #   through a special http path, such as using mod_rewrite.
-    def build_pool(options)
-      hosts = options[:hosts] || ["127.0.0.1"]
-      client_settings = options[:client_settings] || {}
-      timeout = options[:timeout] || 0
+      if sniff_check && values.size > 1
+        raise LogStash::ConfigurationError, "Cannot have multiple values for #{property} in hosts when sniffing is enabled!"
+      end
+
+      uri_value = values.first
+
+      default = nil if default.is_a?(String) && default.empty? # Blanks are as good as nil
+      uri_value = nil if uri_value.is_a?(String) && uri_value.empty?
+
+      if default && uri_value && (default != uri_value)
+        raise LogStash::ConfigurationError, "Explicit value for '#{property}' was declared, but it is different in one of the URLs given! Please make sure your URLs are inline with explicit values. The URLs have the property set to '#{uri_value}', but it was also set to '#{default}' explicitly"
+      end
+
+      uri_value || default
+    end
+
+    def sniffing
+      @options[:sniffing]
+    end
 
-      host_ssl_opt = client_settings[:ssl].nil? ? nil : client_settings[:ssl][:enabled]
-      urls = hosts.map {|host| host_to_url(host, host_ssl_opt, client_settings[:path], client_settings[:parameters])}
+    def user
+      calculate_property(uris, :user, @options[:user], sniffing)
+    end
+
+    def password
+      calculate_property(uris, :password, @options[:password], sniffing)
+    end
+
+    def path
+      calculated = calculate_property(uris, :path, client_settings[:path], sniffing)
+      calculated = "/#{calculated}" if calculated && !calculated.start_with?("/")
+      calculated
+    end
+
+    def scheme
+      explicit_scheme = if ssl_options && ssl_options.has_key?(:enabled)
+                          ssl_options[:enabled] ? 'https' : 'http'
+                        else
+                          nil
+                        end
+      
+      calculated_scheme = calculate_property(uris, :scheme, explicit_scheme, sniffing)
+
+      if calculated_scheme && calculated_scheme !~ /https?/
+        raise LogStash::ConfigurationError, "Bad scheme '#{calculated_scheme}' found should be one of http/https"
+      end
+
+      if calculated_scheme && explicit_scheme && calculated_scheme != explicit_scheme
+        raise LogStash::ConfigurationError, "SSL option was explicitly set to #{ssl_options[:enabled]} but a URL was also declared with a scheme of '#{explicit_scheme}'. Please reconcile this"
+      end
 
+      calculated_scheme # May be nil if explicit_scheme is nil!
+    end
+
+    def port
+      # We don't set the 'default' here because the default is what the user
+      # indicated, so we use an || outside of calculate_property. This lets people
+      # Enter things like foo:123, bar and wind up with foo:123, bar:9200
+      calculate_property(uris, :port, nil, sniffing) || 9200
+    end
+    
+    def uris
+      @options[:hosts]
+    end
+
+    def client_settings
+      @options[:client_settings] || {}
+    end
+
+    def ssl_options
+      client_settings.fetch(:ssl, {})
+    end
+
+    def build_adapter(options)
+      timeout = options[:timeout] || 0
+      
       adapter_options = {
         :socket_timeout => timeout,
         :request_timeout => timeout,
@@ -152,102 +237,58 @@ module LogStash; module Outputs; class ElasticSearch;
         adapter_options[:pool_max_per_route] = client_settings[:pool_max_per_route]
       end
 
-      adapter_options[:ssl] = client_settings[:ssl] if client_settings[:ssl]
-
-      if options[:user]
-        adapter_options[:auth] = {
-          :user => options[:user],
-          :password => options[:password],
-          :eager => true
-        }
-      end
-
+      adapter_options[:ssl] = ssl_options if self.scheme == 'https'
+      
       adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
       adapter = adapter_class.new(@logger, adapter_options)
+    end
+    
+    def build_pool(options)
+      adapter = build_adapter(options)
 
       pool_options = {
-        :sniffing => options[:sniffing],
+        :sniffing => sniffing,
         :sniffer_delay => options[:sniffer_delay],
         :healthcheck_path => options[:healthcheck_path],
-        :resurrect_delay => options[:resurrect_delay]
+        :resurrect_delay => options[:resurrect_delay],
+        :url_normalizer => self.method(:host_to_url)
       }
-
-      ssl_options = options[:client_settings] ? options[:client_settings][:ssl] : {}
-      pool_options[:scheme] = ssl_options && ssl_options[:enabled] ? 'https' : 'http'
-
-      if options[:user]
-        pool_options[:auth] = {:user => options[:user], :password => options[:password]}
-      end
+      pool_options[:scheme] = self.scheme if self.scheme
 
       pool_class = ::LogStash::Outputs::ElasticSearch::HttpClient::Pool
-      pool_class.new(@logger, adapter, urls, pool_options)
+      full_urls = @options[:hosts].map {|h| host_to_url(h) }
+      pool = pool_class.new(@logger, adapter, full_urls, pool_options)
+      pool.start
+      pool
     end
 
-    HOSTNAME_PORT_REGEX=/\A(?<hostname>([A-Za-z0-9\.\-]+)|\[[0-9A-Fa-f\:]+\])(:(?<port>\d+))?\Z/
-    URL_REGEX=/\A#{URI::regexp(['http', 'https'])}\z/
-    # Parse a configuration host to a normalized URL
-    def host_to_url(host, ssl=nil, path=nil, parameters=nil)
-      explicit_scheme = case ssl
-                        when true
-                          "https"
-                        when false
-                          "http"
-                        when nil
-                          nil
-                        else
-                          raise ArgumentError, "Unexpected SSL value!"
-                        end
-
-      # Ensure path starts with a /
-      if path && path[0] != '/'
-        path = "/#{path}"
-      end
-
-      url = nil
-      if host =~ URL_REGEX
-        url = URI.parse(host)
-
-        # Please note that the ssl == nil case is different! If you didn't make an explicit
-        # choice we don't complain!
-        if url.scheme == "http" && ssl == true
-          raise LogStash::ConfigurationError, "You specified a plain 'http' URL '#{host}' but set 'ssl' to true! Aborting!"
-        elsif url.scheme == "https" && ssl == false
-          raise LogStash::ConfigurationError, "You have explicitly disabled SSL but passed in an https URL '#{host}'! Aborting!"
-        end
-
-        url.scheme = explicit_scheme if explicit_scheme
-      elsif (match_results = HOSTNAME_PORT_REGEX.match(host))
-        hostname = match_results["hostname"]
-        port = match_results["port"] || 9200
-        url = URI.parse("#{explicit_scheme || 'http'}://#{hostname}:#{port}")
-      else
-        raise LogStash::ConfigurationError, "Host '#{host}' was specified, but is not valid! Use either a full URL or a hostname:port string!"
-      end
-
-      if path && url.path && url.path != "/" && url.path != ''
-        safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
-        raise LogStash::ConfigurationError, "A path '#{url.path}' has been explicitly specified in the url '#{safe_url}', but you also specified a path of '#{path}'. This is probably a mistake, please remove one setting."
-      end
-
-      if path
-        url.path = path  # The URI library cannot stringify if it holds a nil
-      end
-
-      if parameters
-        query_string = parameters.map { |k,v| "#{k}=#{v}" }.join("&")
-        if query = url.query
-          url.query = "#{query}&#{query_string}"
-        else
-          url.query = query_string
-        end
-      end
-
-      if url.password || url.user
-        raise LogStash::ConfigurationError, "We do not support setting the user password in the URL directly as " +
-          "this may be logged to disk thus leaking credentials. Use the 'user' and 'password' options respectively"
+    def host_to_url(h)
+      # Build a naked URI class to be wrapped in a SafeURI before returning
+      # do NOT log this! It could leak passwords
+      uri_klass = @url_template[:scheme] == 'https' ? URI::HTTPS : URI::HTTP
+      uri = uri_klass.build(@url_template)
+
+      uri.user = h.user || user
+      uri.password = h.password || password
+      uri.host = h.host if h.host
+      uri.port = h.port if h.port
+      uri.path = h.path if !h.path.nil? && !h.path.empty? &&  h.path != "/"
+      uri.query = h.query
+      
+      parameters = client_settings[:parameters]
+      if parameters && !parameters.empty?
+        combined = uri.query ?
+          Hash[URI::decode_www_form(uri.query)].merge(parameters) :
+          parameters
+        query_str = combined.flat_map {|k,v|
+          values = Array(v)
+          values.map {|av| "#{k}=#{av}"}
+        }.join("&")
+        
+        uri.query = query_str
       end
 
-      url
+      ::LogStash::Util::SafeURI.new(uri.normalize)
     end
 
     def template_exists?(name)

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -1,5 +1,4 @@
 require 'manticore'
-require "logstash/outputs/elasticsearch/safe_url"
 
 module LogStash; module Outputs; class ElasticSearch; class HttpClient;
   class ManticoreAdapter
@@ -7,16 +6,32 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
     def initialize(logger, options={})
       @logger = logger
-      @options = options || {}
-      @options[:ssl] = @options[:ssl] || {}
+      options = options.clone || {}
+      options[:ssl] = options[:ssl] || {}
 
       # We manage our own retries directly, so let's disable them here
-      @options[:automatic_retries] = 0
+      options[:automatic_retries] = 0
       # We definitely don't need cookies
-      @options[:cookies] = false
+      options[:cookies] = false
 
-      @request_options = @options[:headers] ? {:headers => @options[:headers]} : {}
-      @manticore = ::Manticore::Client.new(@options)
+      @request_options = options[:headers] ? {:headers => @options[:headers]} : {}
+      
+      if options[:proxy]
+        options[:proxy] = manticore_proxy_hash(options[:proxy])
+      end
+      
+      @manticore = ::Manticore::Client.new(options)
+    end
+    
+    # Transform the proxy option to a hash. Manticore's support for non-hash
+    # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
+    # but this is not yet released
+    def manticore_proxy_hash(proxy_uri)
+      [:scheme, :port, :user, :password, :path].reduce(:host => proxy_uri.host) do |acc,opt|
+        value = proxy_uri.send(opt)
+        acc[opt] = value unless value.nil? || (value.is_a?(String) && value.empty?)
+        acc
+      end
     end
 
     def client
@@ -31,10 +46,27 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def perform_request(url, method, path, params={}, body=nil)
       params = (params || {}).merge @request_options
       params[:body] = body if body
-      # Convert URI object to string
-      url_and_path = path ? (url + path).to_s  : url.to_s
+      
+      request_uri = if path
+                      # Combine the paths using the minimal # of /s
+                      # First, we make sure the path is relative so URI.join does
+                      # the right thing
+                      relative_path = path && path.start_with?("/") ? path[1..-1] : path
+                      # Wrap this with a safe URI defensively against careless handling later
+                      ::LogStash::Util::SafeURI.new(URI.join(url.uri, relative_path))
+                    else
+                      ::LogStash::Util::SafeURI.new(url.uri.clone)
+                    end
+        
+      # We excise auth info from the URL in case manticore itself tries to stick
+      # sensitive data in a thrown exception or log data
+      if request_uri.user
+        params[:auth] = { :user => request_uri.user, :password => request_uri.password, :eager => true }
+        request_uri.user = nil
+        request_uri.password = nil
+      end
 
-      resp = @manticore.send(method.downcase, url_and_path, params)
+      resp = @manticore.send(method.downcase, request_uri.to_s, params)
 
       # Manticore returns lazy responses by default
       # We want to block for our usage, this will wait for the repsonse
@@ -45,8 +77,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # template installation. We might need a better story around this later
       # but for our current purposes this is correct
       if resp.code < 200 || resp.code > 299 && resp.code != 404
-        safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, safe_url + path, resp.body)
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body)
       end
 
       resp