logstash-output-elasticsearch 9.3.0-java → 9.3.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c6ed2c43c43667688e700d6f8db2b714b865a5653c0fc8892a8de479857397d
-  data.tar.gz: bf24c972d847db154b5feb1290341e92b53fdbf5673ea7916d651b77633f9e30
+  metadata.gz: d94e36b467260ee9b470252edef5d67e3d1915a1f1e5d3c7edac1dec879de06d
+  data.tar.gz: 65a256a1b84f2f1ae8de8915423cb3c7e32a756428d2f1110fcaa36e9175181e
 SHA512:
-  metadata.gz: 51939e7522f9370efdc381bf5d981c0efb7a42aa9fc68a873c2281eba9bd4497ec63c40ac951c1b1f2c9c65398073632b13a9870dad05a6d15685dae7c9c6e38
-  data.tar.gz: 18b48cd4103103fcf957718afcaa5f24cfdf429e5ce5949a16a6674ea79035c4801f53049bda2d2a6693469abaf9bb5699547301705961d51159383b1c05a96a
+  metadata.gz: 19f7d8f5ac61a23caf78d0fa919e7a99b35b491a505620f00f3699a5112a38d64c0599f4db543fd9a90e51286393db1ce8e980d5bec0e49ca3d14f7218657def
+  data.tar.gz: ab59d63c93734f63f33d3750bdbd873976f637c57155787c2ebb0aef2459573ba07d74bc85deff9242dfb2bfb144080e56e721117f55fa626beb30b75ddfe126

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.3.1
+  - Fixed issue with escaping index names which was causing writing aliases for ILM to fail [#831](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/831)
+
 ## 9.3.0
   - Adds support for Index Lifecycle Management for Elasticsearch 6.6.0 and above, running with at least a Basic License(Beta) [#805](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/805)
 

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -65,8 +65,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
 
       request_uri = format_url(url, path)
-
-      resp = @manticore.send(method.downcase, request_uri.to_s, params)
+      request_uri_as_string = remove_double_escaping(request_uri.to_s)
+      resp = @manticore.send(method.downcase, request_uri_as_string, params)
 
       # Manticore returns lazy responses by default
       # We want to block for our usage, this will wait for the repsonse
@@ -83,6 +83,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       resp
     end
 
+    # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone
       
@@ -92,23 +93,33 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       request_uri.password = nil
 
       return request_uri.to_s if path_and_query.nil?
-      
+
       parsed_path_and_query = java.net.URI.new(path_and_query)
-      
+
       query = request_uri.query
       parsed_query = parsed_path_and_query.query
-      
+
       new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part|
         part && !part.empty? # Skip empty nil and ""
       end
       
       request_uri.query = new_query_parts.join("&") unless new_query_parts.empty?
 
+      # use `raw_path`` as `path` will unescape any escaped '/' in the path
       request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.raw_path}".gsub(/\/{2,}/, "/")
-
       request_uri
     end
 
+    # Later versions of SafeURI will also escape the '%' sign in an already escaped URI.
+    # (If the path variable is used, it constructs a new java.net.URI object using the multi-arg constructor,
+    # which will escape any '%' characters in the path, as opposed to the single-arg constructor which requires illegal
+    # characters to be already escaped, and will throw otherwise)
+    # The URI needs to have been previously escaped, as it does not play nice with an escaped '/' in the
+    # middle of a URI, as required by date math, treating it as a path separator
+    def remove_double_escaping(url)
+      url.gsub(/%25([0-9A-F]{2})/i, '%\1')
+    end
+
     def close
       @manticore.close
     end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.3.0'
+  s.version         = '9.3.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -30,13 +30,19 @@ module ESHelper
     Time.now.strftime("%Y.%m.%d")
   end
 
-  def mapping_name
+
+  def default_mapping_from_mappings(mappings)
     if ESHelper.es_version_satisfies?(">=7")
-      "_doc"
+      mappings
     else
-      "_default_"
+      mappings["_default_"]
     end
+  end
 
+  def field_properties_from_template(template_name, field)
+    mappings = @es.indices.get_template(name: template_name)[template_name]["mappings"]
+    mapping = default_mapping_from_mappings(mappings)
+    mapping["properties"][field]["properties"]
   end
 
   def routing_field_name

data/spec/integration/outputs/parent_spec.rb

@@ -76,7 +76,15 @@ if ESHelper.es_version_satisfies?(">= 5.6")
 
     shared_examples "a join field based parent indexer" do
       let(:index) { 10.times.collect { rand(10).to_s }.join("") }
-      let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+
+      let(:type) do
+        if ESHelper.es_version_satisfies?('<7')
+          10.times.collect { rand(10).to_s }.join("")
+        else
+          "_doc"
+        end
+      end
+
       let(:event_count) { 10000 + rand(500) }
       let(:parent) { "not_implemented" }
       let(:config) { "not_implemented" }

data/spec/integration/outputs/templates_5x_spec.rb

@@ -82,7 +82,7 @@ if ESHelper.es_version_satisfies?(">= 5")
     end
 
     it "make [geoip][location] a geo_point" do
-      expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"][mapping_name]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
+      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
     end
 
     it "aggregate .keyword results correctly " do

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -102,6 +102,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
       
       it "should add the path correctly" do
         expect(formatted.path).to eq("#{url.path}special_bulk")
+        expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
       end 
       
       it "should add the query parameters correctly" do
@@ -117,6 +118,26 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
         expect(formatted.userinfo).to be_nil
       end
     end
+
+    context 'when uri contains date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
+      end
+    end
+
+    context 'when uri does not contain date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("logstash-0001") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
+      end
+    end
   end
 
   describe "integration specs", :integration => true do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 9.3.0
+  version: 9.3.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-18 00:00:00.000000000 Z
+date: 2019-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement