logstash-output-elasticsearch 5.1.1-java → 5.1.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7460d82e37205595de8bb7fb7f4a7a9cfdeb6817
-  data.tar.gz: 361096568a9c9c0a5fa58d9b495b761f19312b29
+  metadata.gz: d467502b3717a7e37fb897d6efc9fb690d03f045
+  data.tar.gz: 3f73d6fa34b9bc65bce91210bf45dd3fb2e935c1
 SHA512:
-  metadata.gz: ab9ddc3ff94a908c473e79869a3fd2b41682c7facbab54183c40294119b5c23907156a16f677c3181b5f8519670843cc380cd08528b13366f4ce8db3e5837104
-  data.tar.gz: 4eb8391a5fb0df0666a30afabb67ba27e913dfb996e03e0a24edb9e8c73cb8214bfd1e686373385bed0d636034d812207e8930f036d6ff57b001a7c18d3378d7
+  metadata.gz: 7daa79d6457e3fe835cad7603a84016ef1964b4d836fbe766139c3a6a2a0438af0c31c11ca8861150198fb7cec2d362028fe3b379e271d9db7ef3f5d7e2d6ab7
+  data.tar.gz: fae281ebd446f93f4cdbf9785143cfa0d3294d9b9b67d17098b60f8139f4903f0abeddb379e2e25e6406335e8e33291148d3d76e58b91da58581b91503f9a511

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 5.1.2
+- Hide credentials in exceptions and log messages ([#482](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/482))
+- [internal] Remove dependency on longshoreman project
+
 ## 5.1.1
 - Hide user and password from the URL logged during sniffing process.
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -208,7 +208,8 @@ module LogStash; module Outputs; class ElasticSearch;
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         if RETRYABLE_CODES.include?(e.response_code)
-          log_hash = {:code => e.response_code, :url => e.url}
+          safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(e.url)
+          log_hash = {:code => e.response_code, :url => safe_url}
           log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
           @logger.error("Attempted to send a bulk request to elasticsearch but received a bad HTTP response code!", log_hash)
 

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json

@@ -8,7 +8,7 @@
       "_all" : {"enabled" : true, "omit_norms" : true},
       "dynamic_templates" : [ {
         "message_field" : {
-          "match" : "message",
+          "path_match" : "message",
           "match_mapping_type" : "string",
           "mapping" : {
             "type" : "string", "index" : "analyzed", "omit_norms" : true,

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json

@@ -1,5 +1,6 @@
 {
   "template" : "logstash-*",
+  "version" : 50001,
   "settings" : {
     "index.refresh_interval" : "5s"
   },
@@ -8,11 +9,11 @@
       "_all" : {"enabled" : true, "norms" : false},
       "dynamic_templates" : [ {
         "message_field" : {
-          "match" : "message",
+          "path_match" : "message",
           "match_mapping_type" : "string",
           "mapping" : {
-            "type" : "string", "index" : "analyzed", "norms" : false,
-            "fielddata" : { "format" : "disabled" }
+            "type" : "text",
+            "norms" : false
           }
         }
       }, {

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -184,7 +184,8 @@ module LogStash; module Outputs; class ElasticSearch;
       end
 
       if path && url.path && url.path != "/" && url.path != ''
-        raise LogStash::ConfigurationError, "A path '#{url.path}' has been explicitly specified in the url '#{url}', but you also specified a path of '#{path}'. This is probably a mistake, please remove one setting."
+        safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
+        raise LogStash::ConfigurationError, "A path '#{url.path}' has been explicitly specified in the url '#{safe_url}', but you also specified a path of '#{path}'. This is probably a mistake, please remove one setting."
       end
 
       if path

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -194,14 +194,16 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     def resurrect_dead!
       # Try to keep locking granularity low such that we don't affect IO...
       @state_mutex.synchronize { @url_info.select {|url,meta| meta[:dead] } }.each do |url,meta|
+        safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
         begin
-          @logger.info("Checking url #{url} with path #{@healthcheck_path} to see if node resurrected")
+          logger.info("Running health check to see if an Elasticsearch connection is working",
+                      url: safe_url, healthcheck_path: @healthcheck_path)
           perform_request_to_url(url, "HEAD", @healthcheck_path)
           # If no exception was raised it must have succeeded!
-          logger.warn("Resurrected connection to dead ES instance at #{url}")
+          logger.warn("Restored connection to ES instance", :url => safe_url)
           @state_mutex.synchronize { meta[:dead] = false }
         rescue HostUnreachableError => e
-          logger.debug("Attempted to resurrect connection to dead ES instance at #{url}, got an error [#{e.class}] #{e.message}")
+          logger.debug("Attempted to resurrect connection to dead ES instance, but got an error.", url: safe_url, error_type: e.class, error: e.message)
         end
       end
     end
@@ -325,8 +327,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         # In case a sniff happened removing the metadata just before there's nothing to mark
         # This is an extreme edge case, but it can happen!
         return unless meta 
-        logger.warn("Marking url as dead. Last error: [#{error.class}] #{error.message}",
-                    :url => url, :error_message => error.message, :error_class => error.class.name)
+        safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
+        logger.warn("Marking url as dead.", :reason => error.message, :url => safe_url,
+                    :error_message => error.message, :error_class => error.class.name)
         meta[:dead] = true
         meta[:last_error] = error
         meta[:last_errored_at] = Time.now

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '5.1.1'
+  s.version         = '5.1.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -31,7 +31,6 @@ Gem::Specification.new do |s|
   end
 
   s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'longshoreman'
   s.add_development_dependency 'flores'
   # Still used in some specs, we should remove this ASAP
   s.add_development_dependency 'elasticsearch'

data/spec/es_spec_helper.rb

@@ -1,30 +1,10 @@
 require "logstash/devutils/rspec/spec_helper"
 require "ftw"
-require "logstash/plugin"
-require "logstash/json"
-require "stud/try"
-require "longshoreman"
-require "logstash/outputs/elasticsearch"
 require 'elasticsearch'
 
-CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
-CONTAINER_IMAGE = "elasticsearch"
-CONTAINER_TAG = "2.0"
-
-DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
-
 module ESHelper
   def get_host_port
-    addr = DOCKER_INTEGRATION ? Longshoreman.new.get_host_ip : "127.0.0.1"
-    "#{addr}:#{get_port}"
-  end
-
-  def get_port
-    return 9200 unless DOCKER_INTEGRATION
-
-    container = Longshoreman::Container.new
-    container.get(CONTAINER_NAME)
-    container.rport(9200)
+    "127.0.0.1:9200"
   end
 
   def get_client
@@ -32,47 +12,6 @@ module ESHelper
   end
 end
 
-
 RSpec.configure do |config|
   config.include ESHelper
-
-  if DOCKER_INTEGRATION
-    # this :all hook gets run before every describe block that is tagged with :integration => true.
-    config.before(:all, :integration => true) do
-
-
-      # check if container exists already before creating new one.
-      begin
-        ls = Longshoreman::new
-        ls.container.get(CONTAINER_NAME)
-      rescue Docker::Error::NotFoundError
-        scriptDir = File.expand_path File.dirname(__FILE__) + '/fixtures/scripts'
-        Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME, {
-          'Cmd' => [ "-Des.script.inline=on", "-Des.script.indexed=on" ],
-          'HostConfig' => {
-            'Binds' => ["#{scriptDir}:/usr/share/elasticsearch/config/scripts"],
-            'PublishAllPorts' => true
-          }
-        })
-        # TODO(talevy): verify ES is running instead of static timeout
-        sleep 10
-      end
-    end
-
-    # we want to do a final cleanup after all :integration runs,
-    # but we don't want to clean up before the last block.
-    # This is a final blind check to see if the ES docker container is running and
-    # needs to be cleaned up. If no container can be found and/or docker is not
-    # running on the system, we do nothing.
-    config.after(:suite) do
-      # only cleanup docker container if system has docker and the container is running
-      begin
-        ls = Longshoreman::new
-        ls.container.get(CONTAINER_NAME)
-        ls.cleanup
-      rescue Docker::Error::NotFoundError, Excon::Errors::SocketError
-        # do nothing
-      end
-    end
-  end
 end

data/spec/integration/outputs/templates_5x_spec.rb

@@ -27,6 +27,7 @@ describe "index template expected behavior for 5.x", :integration => true, :vers
 
     subject.multi_receive([
       LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
       LogStash::Event.new("somevalue" => 100),
       LogStash::Event.new("somevalue" => 10),
       LogStash::Event.new("somevalue" => 1),
@@ -40,7 +41,7 @@ describe "index template expected behavior for 5.x", :integration => true, :vers
     # Wait or fail until everything's indexed.
     Stud::try(20.times) do
       r = @es.search
-      insist { r["hits"]["total"] } == 7
+      insist { r["hits"]["total"] } == 8
     end
   end
 
@@ -60,11 +61,16 @@ describe "index template expected behavior for 5.x", :integration => true, :vers
     reject { values }.include?(1)
   end
 
-  it "does not create .keyword field for the message field" do
+  it "does not create .keyword field for top-level message field" do
     results = @es.search(:q => "message.keyword:\"sample message here\"")
     insist { results["hits"]["total"] } == 0
   end
 
+  it "creates .keyword field for nested message fields" do
+    results = @es.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+    insist { results["hits"]["total"] } == 1
+  end
+
   it "creates .keyword field from any string field which is not_analyzed" do
     results = @es.search(:q => "country.keyword:\"us\"")
     insist { results["hits"]["total"] } == 1

data/spec/integration/outputs/templates_spec.rb

@@ -26,6 +26,7 @@ describe "index template expected behavior", :integration => true, :version_less
 
     subject.multi_receive([
       LogStash::Event.new("message" => "sample message here"),
+      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
       LogStash::Event.new("somevalue" => 100),
       LogStash::Event.new("somevalue" => 10),
       LogStash::Event.new("somevalue" => 1),
@@ -39,7 +40,7 @@ describe "index template expected behavior", :integration => true, :version_less
     # Wait or fail until everything's indexed.
     Stud::try(20.times) do
       r = @es.search
-      insist { r["hits"]["total"] } == 7
+      insist { r["hits"]["total"] } == 8
     end
   end
 
@@ -64,6 +65,11 @@ describe "index template expected behavior", :integration => true, :version_less
     insist { results["hits"]["total"] } == 0
   end
 
+  it "creates .raw field for nested message fields" do
+    results = @es.search(:q => "somemessage.message.raw:\"sample nested message here\"")
+    insist { results["hits"]["total"] } == 1
+  end
+
   it "creates .raw field from any string field which is not_analyzed" do
     results = @es.search(:q => "country.raw:\"us\"")
     insist { results["hits"]["total"] } == 1

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -48,7 +48,7 @@ describe LogStash::Outputs::ElasticSearch::TemplateManager do
       end
     end
     context "elasticsearch 5.x" do
-      it "chooses the 2x template" do
+      it "chooses the 5x template" do
         expect(described_class.default_template_path("5")).to match(/elasticsearch-template-es5x.json/)
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.1.1
+  version: 5.1.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-13 00:00:00.000000000 Z
+date: 2016-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -126,20 +126,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: longshoreman
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: