logstash-output-elasticsearch 9.0.2-java → 9.0.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 910353088f57a02bea121f9f2c92e27d27664372dbf6b934e256103178c093cd
-  data.tar.gz: a5ba64f0511129dbc9fa8890457f25317e7541aecaf4e694f4f1b835143ce999
+  metadata.gz: ae7148b3b08c44548f172bc06c88ff48b94809ef756aeef21d4597efb2d50716
+  data.tar.gz: a6c62363e0d1f953cfe9ee9ef8ceed192f7d54f0e3421f2c403f1c110a11a1ed
 SHA512:
-  metadata.gz: 03a87ed4822764503c6d0d5269ac5c19df9b0d5566659f0ca0df6ced8abd4f6aebd1e384cd6f2fa829787cd864730b32a90235378b88954eba25ebceff240228
-  data.tar.gz: 357011f25a5e41559e995bf35cbb25de6d0ef4c5db0524994dcb24bd2d8300e36068793a403b40e3fc5face5495a9e9963546146c89870c0d219c269864f8afb
+  metadata.gz: 420c2f66334764c1693936ca462cb45e8b196dec449ae7521373ba44c5521e6f82b76f9d3ac02a950dc597149b671701f16e2226f0070a8196c8946a2bdc9f92
+  data.tar.gz: 0f30be7f309a6734f6f0e050b8ff725db3ce71a7ce0c617997f35b0ec013410c7ca56991c59978ec147da8d87cb598d214272e7325ce84145257341fdfd4568c

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.0.3
+  - Ignore master-only nodes when using sniffing
+
 ## 9.0.2
   - Ignore event's type field for the purpose of setting document `_type` if cluster is es 6.x or above
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -519,10 +519,8 @@ if enabled, script is in charge of creating non-existent document (scripted upda
   * Default value is `false`
 
 This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
-Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
-this with master nodes, you probably want to disable HTTP on them by setting
-`http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
-manually enter multiple Elasticsearch hosts using the `hosts` parameter.
+For Elasticsearch 1.x and 2.x any nodes with `http.enabled` (on by default) will be added to the hosts list, including master-only nodes!
+For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will be added to the hosts list, excluding master-only nodes.
 
 [id="plugins-{type}s-{plugin}-sniffing_delay"]
 ===== `sniffing_delay` 
@@ -635,7 +633,7 @@ Set the truststore password
   * Value type is <<string,string>>
   * Default value is `""`
 
-Set upsert content for update mode.s
+Set upsert content for update mode.
 Create a new document with this parameter as json string if `document_id` doesn't exists
 
 [id="plugins-{type}s-{plugin}-user"]

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json

@@ -0,0 +1,45 @@
+{
+  "template" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_doc" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date"},
+        "@version": { "type": "keyword"},
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -186,6 +186,9 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     
     def sniff_5x_and_above(nodes)
       nodes.map do |id,info|
+        # Skip master-only nodes
+        next if info["roles"] && info["roles"] == ["master"]
+
         if info["http"]
           uri = LogStash::Util::SafeURI.new(info["http"]["publish_address"])
         end
@@ -247,7 +250,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
             meta[:version] = es_version
             major = major_version(es_version)
             if !@maximum_seen_major_version
-              @logger.info("ES Output version determined", :es_version => @maximum_seen_major_version)
+              @logger.info("ES Output version determined", :es_version => major)
               set_new_major_version(major)
             elsif major > @maximum_seen_major_version
               @logger.warn("Detected a node with a higher major version than previously observed. This could be the result of an elasticsearch cluster upgrade.", :previous_major => @maximum_seen_major_version, :new_major => major, :node_url => url)

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.0.2'
+  s.version         = '9.0.3'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/_nodes/2x_1x.json

@@ -0,0 +1,27 @@
+{
+  "cluster_name" : "dev",
+  "nodes" : {
+    "Ur_68iBvTlm7Xr1HgSsh6w" : {
+      "name" : "dev-es-master01",
+      "transport_address" : "http://localhost:9200",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1",
+      "version" : "2.4.6",
+      "build" : "5376dca"
+    },
+    "sari4do3RG-mgh2CIZeHwA" : {
+      "name" : "dev-es-data01",
+      "transport_address" : "http://localhost:9201",
+      "host" : "127.0.0.1",
+      "ip" : "127.0.0.1",
+      "version" : "2.4.6",
+      "build" : "5376dca",
+      "http_address" : "127.0.0.1:9201",
+      "http" : {
+        "bound_address" : [ "[::1]:9201", "127.0.0.1:9201" ],
+        "publish_address" : "127.0.0.1:9201",
+        "max_content_length_in_bytes" : 104857600
+      }
+    }
+  }
+}

data/spec/fixtures/_nodes/5x_and_above.json

@@ -0,0 +1,81 @@
+{
+  "_nodes" : {
+    "total" : 3,
+    "successful" : 3,
+    "failed" : 0
+  },
+  "cluster_name" : "dev",
+  "nodes" : {
+    "Ur_68iBvTlm7Xr1HgSsh6w" : {
+      "name" : "dev-es-master01",
+      "transport_address" : "http://localhost:9200",
+      "host" : "localhost",
+      "ip" : "127.0.0.1",
+      "version" : "5.5.1",
+      "build_hash" : "19c13d0",
+      "roles" : [
+        "master"
+      ],
+      "http" : {
+        "bound_address" : [
+          "[::]:9200"
+        ],
+        "publish_address" : "http://localhost:9200",
+        "max_content_length_in_bytes" : 104857600
+      }
+    },
+    "sari4do3RG-mgh2CIZeHwA" : {
+      "name" : "dev-es-data01",
+      "transport_address" : "http://localhost:9201",
+      "host" : "localhost",
+      "ip" : "127.0.0.1",
+      "version" : "5.5.1",
+      "build_hash" : "19c13d0",
+      "roles" : [
+        "data"
+      ],
+      "http" : {
+        "bound_address" : [
+          "[::]:9200"
+        ],
+        "publish_address" : "http://localhost:9201",
+        "max_content_length_in_bytes" : 104857600
+      }
+    },
+    "Rjy1WL66RHm4fyzXA8PCGQ" : {
+      "name" : "dev-es-datamaster01",
+      "transport_address" : "http://localhost:9202",
+      "host" : "localhost",
+      "ip" : "127.0.0.1",
+      "version" : "5.5.1",
+      "build_hash" : "19c13d0",
+      "roles" : [
+        "data",
+        "master"
+      ],
+      "http" : {
+        "bound_address" : [
+          "[::]:9200"
+        ],
+        "publish_address" : "http://localhost:9202",
+        "max_content_length_in_bytes" : 104857600
+      }
+    },
+    "OguP_obcT_S9JYNB8SKKgQ" : {
+      "name" : "dev-es-coordinator01",
+      "transport_address" : "http://localhost:9203",
+      "host" : "localhost",
+      "ip" : "127.0.0.1",
+      "version" : "5.5.1",
+      "build_hash" : "19c13d0",
+      "roles" : [ ],
+      "http" : {
+        "bound_address" : [
+          "[::]:9200"
+        ],
+        "publish_address" : "http://localhost:9203",
+        "max_content_length_in_bytes" : 104857600
+      }
+    }
+  }
+}

data/spec/integration/outputs/sniffer_spec.rb

@@ -10,38 +10,91 @@ describe "pool sniffer", :integration => true do
   let(:options) { {:resurrect_delay => 2, :url_normalizer => proc {|u| u}} } # Shorten the delay a bit to speed up tests
 
   subject { LogStash::Outputs::ElasticSearch::HttpClient::Pool.new(logger, adapter, initial_urls, options) }
-  
-  before do
-    subject.start
-  end
-  
-  shared_examples("sniff parsing") do |check_exact|
-    it "should execute a sniff without error" do
-      expect { subject.check_sniff }.not_to raise_error
-    end
 
-    it "should return the correct sniff URL list" do
-      uris = subject.check_sniff
-      
-      # ES 1.x returned the public hostname by default. This is hard to approximate
-      # so for ES1.x we don't check the *exact* hostname
-      if check_exact
-        expect(uris).to include(::LogStash::Util::SafeURI.new("//#{get_host_port}"))
-      else
+  describe("Simple sniff parsing")  do
+    before(:each) { subject.start }
+
+    context "with single node" do
+      it "should execute a sniff without error" do
+        expect { subject.check_sniff }.not_to raise_error
+      end
+
+      it "should return single sniff URL" do
+        uris = subject.check_sniff
+
         expect(uris.size).to eq(1)
       end
+
+      it "should return the correct sniff URL" do
+        if ESHelper.es_version_satisfies?(">= 2")
+          # We do a more thorough check on these versions because we can more reliably guess the ip
+          uris = subject.check_sniff
+
+          expect(uris).to include(::LogStash::Util::SafeURI.new("//#{get_host_port}"))
+        else
+          # ES 1.x returned the public hostname by default. This is hard to approximate
+          # so for ES1.x we don't check the *exact* hostname
+          skip
+        end
+      end
     end
   end
-  
-  describe("Simple sniff parsing")  do
-    include_examples("sniff parsing", false)
+
+  if ESHelper.es_version_satisfies?("<= 2")
+    describe("Complex sniff parsing ES 2x/1x") do
+      before(:each) do
+        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/2x_1x.json"))
+        allow(subject).to receive(:perform_request).and_return([nil, { version: "2.0" }, response_double])
+        subject.start
+      end
+
+      context "with multiple nodes but single http-enabled data node" do
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return one sniff URL" do
+          uris = subject.check_sniff
+
+          expect(uris.size).to eq(1)
+        end
+
+        it "should return the correct sniff URL" do
+          if ESHelper.es_version_satisfies?(">= 2")
+            # We do a more thorough check on these versions because we can more reliably guess the ip
+            uris = subject.check_sniff
+
+            expect(uris).to include(::LogStash::Util::SafeURI.new("http://localhost:9201"))
+          else
+            # ES 1.x returned the public hostname by default. This is hard to approximate
+            # so for ES1.x we don't check the *exact* hostname
+            skip
+          end
+        end
+      end
+    end
   end
-  
-  # We do a more thorough check on these versions because we can more reliably guess the ip
 
-  if ESHelper.es_version_satisfies?(">= 2")
-    describe("Complex sniff parsing ES 6x/5x/2x") do
-      include_examples("sniff parsing", true)
+  if ESHelper.es_version_satisfies?(">= 5")
+    describe("Complex sniff parsing ES 6x/5x") do
+      before(:each) do
+        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/5x_and_above.json"))
+        allow(subject).to receive(:perform_request).and_return([nil, { version: "5.0" }, response_double])
+        subject.start
+      end
+
+      context "with mixed master-only, data-only, and data + master nodes" do
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return the correct sniff URLs" do
+          # ie. without the master-only node
+          uris = subject.check_sniff
+
+          expect(uris).to include(::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202"), ::LogStash::Util::SafeURI.new("http://localhost:9203"))
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 9.0.2
+  version: 9.0.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-30 00:00:00.000000000 Z
+date: 2018-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -217,6 +217,7 @@ files:
 - lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json
 - lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json
 - lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json
+- lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
 - lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
 - lib/logstash/outputs/elasticsearch/http_client/pool.rb
@@ -224,6 +225,8 @@ files:
 - lib/logstash/outputs/elasticsearch/template_manager.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
+- spec/fixtures/_nodes/2x_1x.json
+- spec/fixtures/_nodes/5x_and_above.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/groovy/scripted_update.groovy
@@ -288,6 +291,8 @@ specification_version: 4
 summary: Stores logs in Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
+- spec/fixtures/_nodes/2x_1x.json
+- spec/fixtures/_nodes/5x_and_above.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/groovy/scripted_update.groovy