logstash-output-elasticsearch 9.3.1-java → 9.3.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d94e36b467260ee9b470252edef5d67e3d1915a1f1e5d3c7edac1dec879de06d
-  data.tar.gz: 65a256a1b84f2f1ae8de8915423cb3c7e32a756428d2f1110fcaa36e9175181e
+  metadata.gz: 4ae86a048dbbbc543991c0f0940002f81c13ef3619a3ab0dcfbd73030b4992be
+  data.tar.gz: 8ccb17687bc962888d4a0f2eb94d62a29f27ec7460078deb98f4a789ef9ac13f
 SHA512:
-  metadata.gz: 19f7d8f5ac61a23caf78d0fa919e7a99b35b491a505620f00f3699a5112a38d64c0599f4db543fd9a90e51286393db1ce8e980d5bec0e49ca3d14f7218657def
-  data.tar.gz: ab59d63c93734f63f33d3750bdbd873976f637c57155787c2ebb0aef2459573ba07d74bc85deff9242dfb2bfb144080e56e721117f55fa626beb30b75ddfe126
+  metadata.gz: 89be3b3bf2a12d130ae7d6c8da1f9961d827b12814901c180a9caa966ca15bb7b8ae2a1df0b79906edd48edaca7888ed9dd8f6c132a5b52170641a777fcd0e54
+  data.tar.gz: 890f65b47189d95c7087095e7f1a0cfad288499428de0002cd260f40a67788b43a9f4b7c6a643d9428ee25cd5e326fe336191bda6e674f5eea264b21f413c97f

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.3.2
+  - Fixed sniffing support for 7.x [#827](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/827)
+
 ## 9.3.1
   - Fixed issue with escaping index names which was causing writing aliases for ILM to fail [#831](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/831)
 

data/docs/index.asciidoc

@@ -1,6 +1,6 @@
 :plugin: elasticsearch
 :type: output
-:default_codec: plain
+:no_codec:
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -98,6 +98,9 @@ happens, the problem is logged as a warning, and the event is dropped. See
 
 beta[]
 
+[NOTE]
+The Index Lifecycle Management feature requires plugin version `9.3.1` or higher.
+
 [NOTE]
 This feature requires an Elasticsearch instance of 6.6.0 or higher with at least a Basic license
 
@@ -814,4 +817,4 @@ See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-in
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]
 
-:default_codec!:
+:no_codec!:

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json

@@ -1,44 +1,42 @@
 {
-  "template" : "logstash-*",
+  "index_patterns" : "logstash-*",
   "version" : 60001,
   "settings" : {
     "index.refresh_interval" : "5s",
     "number_of_shards": 1
   },
   "mappings" : {
-    "_doc" : {
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text",
-            "norms" : false
-          }
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
         }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text", "norms" : false,
-            "fields" : {
-              "keyword" : { "type": "keyword", "ignore_above": 256 }
-            }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
           }
         }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date"},
-        "@version": { "type": "keyword"},
-        "geoip"  : {
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip" },
-            "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "half_float" },
-            "longitude" : { "type" : "half_float" }
-          }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
         }
       }
     }

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -158,7 +158,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     end
 
     ES1_SNIFF_RE_URL  = /\[([^\/]*)?\/?([^:]*):([0-9]+)\]/
-    ES2_SNIFF_RE_URL  = /([^\/]*)?\/?([^:]*):([0-9]+)/
+    ES2_AND_ABOVE_SNIFF_RE_URL  = /([^\/]*)?\/?([^:]*):([0-9]+)/
     # Sniffs and returns the results. Does not update internal URLs!
     def check_sniff
       _, url_meta, resp = perform_request(:get, @sniffing_path)
@@ -189,13 +189,19 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       nodes.map do |id,info|
         # Skip master-only nodes
         next if info["roles"] && info["roles"] == ["master"]
-
-        if info["http"]
-          uri = LogStash::Util::SafeURI.new(info["http"]["publish_address"])
-        end
+        address_str_to_uri(info["http"]["publish_address"]) if info["http"]
       end.compact
     end
-    
+
+    def address_str_to_uri(addr_str)
+      matches = addr_str.match(ES1_SNIFF_RE_URL) || addr_str.match(ES2_AND_ABOVE_SNIFF_RE_URL)
+      if matches
+        host = matches[1].empty? ? matches[2] : matches[1]
+        ::LogStash::Util::SafeURI.new("#{host}:#{matches[3]}")
+      end
+    end
+
+
     def sniff_2x_1x(nodes)
       nodes.map do |id,info|
         # TODO Make sure this works with shield. Does that listed
@@ -210,13 +216,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         # with master + data + client enabled, so we allow that
         attributes = info['attributes']
         next if attributes && attributes['data'] == 'false'
-
-        matches = addr_str.match(ES1_SNIFF_RE_URL) || addr_str.match(ES2_SNIFF_RE_URL)
-        if matches
-          host = matches[1].empty? ? matches[2] : matches[1]
-          port = matches[3]
-          ::LogStash::Util::SafeURI.new("#{host}:#{port}")
-        end
+        address_str_to_uri(addr_str)
       end.compact
     end
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.3.1'
+  s.version         = '9.3.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/_nodes/{5x_and_above.json → 5x_6x.json}

@@ -20,7 +20,7 @@
         "bound_address" : [
           "[::]:9200"
         ],
-        "publish_address" : "http://localhost:9200",
+        "publish_address" : "127.0.0.1:9200",
         "max_content_length_in_bytes" : 104857600
       }
     },
@@ -38,7 +38,7 @@
         "bound_address" : [
           "[::]:9200"
         ],
-        "publish_address" : "http://localhost:9201",
+        "publish_address" : "127.0.0.1:9201",
         "max_content_length_in_bytes" : 104857600
       }
     },
@@ -57,7 +57,7 @@
         "bound_address" : [
           "[::]:9200"
         ],
-        "publish_address" : "http://localhost:9202",
+        "publish_address" : "127.0.0.1:9202",
         "max_content_length_in_bytes" : 104857600
       }
     },
@@ -73,7 +73,7 @@
         "bound_address" : [
           "[::]:9200"
         ],
-        "publish_address" : "http://localhost:9203",
+        "publish_address" : "127.0.0.1:9203",
         "max_content_length_in_bytes" : 104857600
       }
     }

data/spec/fixtures/_nodes/7x.json

@@ -0,0 +1,92 @@
+{
+  "_nodes" : {
+    "total" : 3,
+    "successful" : 3,
+    "failed" : 0
+  },
+  "cluster_name" : "elasticsearch",
+  "nodes" : {
+    "kVPTh7ZvSgWmTRMy-4YExQ" : {
+      "name" : "kVPTh7Z",
+      "transport_address" : "127.0.0.1:9300",
+      "host" : "dev-master",
+      "ip" : "127.0.0.1",
+      "version" : "7.0.0",
+      "build_flavor" : "default",
+      "build_type" : "tar",
+      "build_hash" : "b0e7036",
+      "roles" : [
+        "master"
+      ],
+      "attributes" : {
+        "ml.machine_memory" : "17179869184",
+        "xpack.installed" : "true",
+        "ml.max_open_jobs" : "20",
+        "ml.enabled" : "true"
+      },
+      "http" : {
+        "bound_address" : [
+          "127.0.0.1:9200",
+          "[::1]:9200"
+        ],
+        "publish_address" : "dev-master/127.0.0.1:9200",
+        "max_content_length_in_bytes" : 104857600
+      }
+    },
+    "J47OFlfpSHGFwRJSF2hbcg" : {
+      "name" : "J47OFlf",
+      "transport_address" : "127.0.0.1:9301",
+      "host" : "dev-masterdata",
+      "ip" : "127.0.0.1",
+      "version" : "7.0.0",
+      "build_flavor" : "default",
+      "build_type" : "tar",
+      "build_hash" : "b0e7036",
+      "roles" : [
+        "master",
+        "data"
+      ],
+      "attributes" : {
+        "ml.machine_memory" : "17179869184",
+        "ml.max_open_jobs" : "20",
+        "xpack.installed" : "true",
+        "ml.enabled" : "true"
+      },
+      "http" : {
+        "bound_address" : [
+          "127.0.0.1:9201",
+          "[::1]:9201"
+        ],
+        "publish_address" : "dev-masterdata/127.0.0.1:9201",
+        "max_content_length_in_bytes" : 104857600
+      }
+    },
+    "pDYE99f0QmutVb8gvsf-yw" : {
+      "name" : "pDYE99f",
+      "transport_address" : "127.0.0.1:9302",
+      "host" : "dev-data",
+      "ip" : "127.0.0.1",
+      "version" : "7.0.0",
+      "build_flavor" : "default",
+      "build_type" : "tar",
+      "build_hash" : "b0e7036",
+      "roles" : [
+        "data"
+      ],
+      "attributes" : {
+        "ml.machine_memory" : "17179869184",
+        "ml.max_open_jobs" : "20",
+        "xpack.installed" : "true",
+        "ml.enabled" : "true"
+      },
+      "http" : {
+        "bound_address" : [
+          "127.0.0.1:9202",
+          "[::1]:9202"
+        ],
+        "publish_address" : "dev-data/127.0.0.1:9202",
+        "max_content_length_in_bytes" : 104857600
+      }
+    }
+  }
+}

data/spec/integration/outputs/sniffer_spec.rb

@@ -32,14 +32,14 @@ describe "pool sniffer", :integration => true do
       end
 
       it "should return the correct sniff URL" do
-        if ESHelper.es_version_satisfies?(">= 2")
+        if ESHelper.es_version_satisfies?(">= 2", "<7")
           # We do a more thorough check on these versions because we can more reliably guess the ip
           uris = subject.check_sniff
 
           expect(uris).to include(::LogStash::Util::SafeURI.new("//#{get_host_port}"))
         else
-          # ES 1.x returned the public hostname by default. This is hard to approximate
-          # so for ES1.x we don't check the *exact* hostname
+          # ES 1.x (and ES 7.x) returned the public hostname by default. This is hard to approximate
+          # so for ES1.x and 7.x we don't check the *exact* hostname
           skip
         end
       end
@@ -81,10 +81,33 @@ describe "pool sniffer", :integration => true do
     end
   end
 
+
+  if ESHelper.es_version_satisfies?(">= 7")
+    describe("Complex sniff parsing ES 7x") do
+      before(:each) do
+        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/7x.json"))
+        allow(subject).to receive(:perform_request).and_return([nil, { version: "7.0" }, response_double])
+        subject.start
+      end
+
+      context "with mixed master-only, data-only, and data + master nodes" do
+        it "should execute a sniff without error" do
+          expect { subject.check_sniff }.not_to raise_error
+        end
+
+        it "should return the correct sniff URLs" do
+          # ie. with the master-only node, and with the node name correctly set.
+          uris = subject.check_sniff
+
+          expect(uris).to include(::LogStash::Util::SafeURI.new("//dev-masterdata:9201"), ::LogStash::Util::SafeURI.new("//dev-data:9202"))
+        end
+      end
+    end
+  end
   if ESHelper.es_version_satisfies?(">= 5")
     describe("Complex sniff parsing ES 6x/5x") do
       before(:each) do
-        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/5x_and_above.json"))
+        response_double = double("_nodes/http", body: File.read("spec/fixtures/_nodes/5x_6x.json"))
         allow(subject).to receive(:perform_request).and_return([nil, { version: "5.0" }, response_double])
         subject.start
       end
@@ -98,7 +121,7 @@ describe "pool sniffer", :integration => true do
           # ie. without the master-only node
           uris = subject.check_sniff
 
-          expect(uris).to include(::LogStash::Util::SafeURI.new("http://localhost:9201"), ::LogStash::Util::SafeURI.new("http://localhost:9202"), ::LogStash::Util::SafeURI.new("http://localhost:9203"))
+          expect(uris).to include(::LogStash::Util::SafeURI.new("//127.0.0.1:9201"), ::LogStash::Util::SafeURI.new("//127.0.0.1:9202"), ::LogStash::Util::SafeURI.new("//127.0.0.1:9203"))
         end
       end
     end

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -76,6 +76,48 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
   end
 
+  describe 'resolving the address from Elasticsearch node info' do
+    let(:host) { "node.elastic.co"}
+    let(:ip_address) { "192.168.1.0"}
+    let(:port) { 9200 }
+
+    context 'in Elasticsearch 1.x format' do
+      context 'with host and ip address' do
+        let(:publish_address) { "inet[#{host}/#{ip_address}:#{port}]"}
+        it 'should correctly extract the host' do
+          expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
+        end
+      end
+      context 'with ip address' do
+        let(:publish_address) { "inet[/#{ip_address}:#{port}]"}
+        it 'should correctly extract the ip address' do
+          expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
+        end
+      end
+    end
+
+    context 'in Elasticsearch 2.x-6.x format' do
+      let(:publish_address) { "#{ip_address}:#{port}"}
+      it 'should correctly extract the ip address' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("//#{ip_address}:#{port}"))
+      end
+    end
+
+    context 'in Elasticsearch 7.x'
+    context 'with host and ip address' do
+      let(:publish_address) { "#{host}/#{ip_address}:#{port}"}
+      it 'should correctly extract the host' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{host}:#{port}"))
+      end
+    end
+    context 'with ip address' do
+      let(:publish_address) { "#{ip_address}:#{port}"}
+      it 'should correctly extract the ip address' do
+        expect(subject.address_str_to_uri(publish_address)).to eq (LogStash::Util::SafeURI.new("#{ip_address}:#{port}"))
+      end
+    end
+  end
+
   describe "the sniffer" do
     before(:each) { subject.start }
     it "should not start the sniffer by default" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 9.3.1
+  version: 9.3.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-29 00:00:00.000000000 Z
+date: 2019-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -172,7 +172,8 @@ files:
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
 - spec/fixtures/_nodes/2x_1x.json
-- spec/fixtures/_nodes/5x_and_above.json
+- spec/fixtures/_nodes/5x_6x.json
+- spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/groovy/scripted_update.groovy
@@ -248,7 +249,8 @@ summary: Stores logs in Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
 - spec/fixtures/_nodes/2x_1x.json
-- spec/fixtures/_nodes/5x_and_above.json
+- spec/fixtures/_nodes/5x_6x.json
+- spec/fixtures/_nodes/7x.json
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
 - spec/fixtures/scripts/groovy/scripted_update.groovy