logstash-output-elasticsearch 8.0.1-java → 8.1.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c67a2d242828906ff0f1a9b5d90c09c7db30b7b0
-  data.tar.gz: 64da8c0ac68f583f4ba13fbb7e42372aa349e442
+  metadata.gz: fbfa89ded2964b66fde5e29cd3155d8f2da08e03
+  data.tar.gz: 3c603e14218bc92384c4112527d38c834ff7fb3e
 SHA512:
-  metadata.gz: f8b439be4170362cdbebf0aff83a98d49a8996826de5a31946d772ba4e1633420fc06f3c0e68d7b4f3f00e62e5dede53b8fd1c7b798a1822973605489058cc8b
-  data.tar.gz: 3b204be36010c921c014cfa5b3ed867d4b13ae2d40271437d8b4caad04763003182f23a7e79f4334527df2a8179af8ec2a0478e0fd9bc29be086c558fc9e24c5
+  metadata.gz: aea0668c8ac704c1db41f01f62bdf84a173776b8be7a55e04a6de6898121a3f89a3b94be2c7a94b1d75c17407926bb8c033627dcb1507a1fcbae4d1b04aa16c8
+  data.tar.gz: 23cce8e779627097c33cf6ebb59106f6353cd40fa9d786f67b74fa42655738646104dfdd1a5ddfc27e4609b6187fec3018eb540e0848c5862129662d5686fd59

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 8.1.1
+  - Retry all non-200 responses of the bulk API indefinitely
+  - Improve documentation on retry codes
+
+## 8.1.0
+  - Support Elasticsearch 6.x join field type
+## 8.0.2
+  - Fix bug where logging errors for bad response codes would raise an unhandled exception
+
 ## 8.0.1
   - Fix some documentation issues
 

data/docs/index.asciidoc

@@ -55,17 +55,18 @@ the new template is installed.
 
 ==== Retry Policy
 
-The retry policy has changed significantly in the 2.2.0 release.
+The retry policy has changed significantly in the 8.1.1 release.
 This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
-either partial or total failures.
+either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP
+request are handled differently than error codes for individual documents.
 
-The following errors are retried infinitely:
+HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely.
 
-- Network errors (inability to connect)
-- 429 (Too many requests) and
-- 503 (Service unavailable) errors
+The following document errors are handled as follows:
+- 400 and 404 errors are sent to the DLQ if enabled. If a DLQ is not enabled a log message will be emitted and the event will be dropped.
+- 409 errors (conflict) are logged as a warning and dropped. 
 
-NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
+Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
 It is more performant for Elasticsearch to retry these exceptions than this plugin.
 
 ==== Batch Sizes ====

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -4,14 +4,10 @@ module LogStash; module Outputs; class ElasticSearch;
   module Common
     attr_reader :client, :hosts
 
-    # These are codes for temporary recoverable conditions
-    # 429 just means that ES has too much traffic ATM
-    # 503 means it , or a proxy is temporarily unavailable
-    RETRYABLE_CODES = [429, 503]
-
-    DLQ_CODES = [400, 404]
-    SUCCESS_CODES = [200, 201]
-    CONFLICT_CODE = 409
+    # These codes apply to documents, not at the request level
+    DOC_DLQ_CODES = [400, 404]
+    DOC_SUCCESS_CODES = [200, 201]
+    DOC_CONFLICT_CODE = 409
 
     # When you use external versioning, you are communicating that you want
     # to ignore conflicts. More obviously, since an external version is a 
@@ -134,12 +130,12 @@ module LogStash; module Outputs; class ElasticSearch;
         # - For 409, we log and drop. there is nothing we can do
         # - For a mapping error, we send to dead letter queue for a human to intervene at a later point.
         # - For everything else there's mastercard. Yep, and we retry indefinitely. This should fix #572 and other transient network issues
-        if SUCCESS_CODES.include?(status)
+        if DOC_SUCCESS_CODES.include?(status)
           next
-        elsif CONFLICT_CODE == status
+        elsif DOC_CONFLICT_CODE == status
           @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
           next
-        elsif DLQ_CODES.include?(status)
+        elsif DOC_DLQ_CODES.include?(status)
           action_event = action[2]
           # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
           if @dlq_writer
@@ -174,8 +170,15 @@ module LogStash; module Outputs; class ElasticSearch;
         params[:pipeline] = event.sprintf(@pipeline)
       end
 
-     if @parent
-        params[:parent] = event.sprintf(@parent)
+      if @parent
+        if @join_field
+          join_value = event.get(@join_field)
+          parent_value = event.sprintf(@parent)
+          event.set(@join_field, { "name" => join_value, "parent" => parent_value })
+          params[:_routing] = event.sprintf(@parent)
+        else
+          params[:parent] = event.sprintf(@parent)
+        end
       end
 
       if @action == 'update'
@@ -244,29 +247,22 @@ module LogStash; module Outputs; class ElasticSearch;
         sleep_interval = next_sleep_interval(sleep_interval)
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
-        if RETRYABLE_CODES.include?(e.response_code)
-          log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
-          log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
-          message = "Encountered a retryable error. Will Retry with exponential backoff "
-
-          # We treat 429s as a special case because these really aren't errors, but
-          # rather just ES telling us to back off a bit, which we do.
-          # The other retryable code is 503, which are true errors
-          # Even though we retry the user should be made aware of these
-          if e.response_code == 429
-            logger.debug(message, log_hash)
-          else
-            logger.error(message, log_hash)
-          end
-
-          sleep_interval = sleep_for_interval(sleep_interval)
-          retry
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
+        log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
+        message = "Encountered a retryable error. Will Retry with exponential backoff "
+
+        # We treat 429s as a special case because these really aren't errors, but
+        # rather just ES telling us to back off a bit, which we do.
+        # The other retryable code is 503, which are true errors
+        # Even though we retry the user should be made aware of these
+        if e.response_code == 429
+          logger.debug(message, log_hash)
         else
-          log_hash = {:code => e.response_code, 
-                      :response_body => e.response_body}
-          log_hash[:request_body] = e.request_body if @logger.debug?
-          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", log_hash)
+          logger.error(message, log_hash)
         end
+
+        sleep_interval = sleep_for_interval(sleep_interval)
+        retry
       rescue => e
         # Stuff that should never happen
         # For all other errors print out full connection issues
@@ -279,7 +275,6 @@ module LogStash; module Outputs; class ElasticSearch;
 
         @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
 
-        # We retry until there are no errors! Errors should all go to the retry queue
         sleep_interval = sleep_for_interval(sleep_interval)
         retry unless @stopping.true?
       end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -78,6 +78,9 @@ module LogStash; module Outputs; class ElasticSearch
       # This can be dynamic using the `%{foo}` syntax.
       mod.config :parent, :validate => :string, :default => nil
 
+      # For child documents, name of the join field
+      mod.config :join_field, :validate => :string, :default => nil
+
       # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
       # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
       #     `"127.0.0.1"`

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -142,6 +142,13 @@ module LogStash; module Outputs; class ElasticSearch;
         body_stream.truncate(0)
         body_stream.seek(0)
       end
+
+      if response.code != 200
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+          response.code, @bulk_path, body_stream.to_s, response.body
+        )
+      end
+
       LogStash::Json.load(response.body)
     end
 

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -66,7 +66,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
       request_uri = format_url(url, path)
 
-      resp = @manticore.send(method.downcase, request_uri, params)
+      resp = @manticore.send(method.downcase, request_uri.to_s, params)
 
       # Manticore returns lazy responses by default
       # We want to block for our usage, this will wait for the repsonse
@@ -106,7 +106,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       
       request_uri.path = "#{request_uri.path}/#{parsed_path_and_query.path}".gsub(/\/{2,}/, "/")
         
-      request_uri.to_s
+      request_uri
     end
 
     def close

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '8.0.1'
+  s.version         = '8.1.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/parent_spec.rb

@@ -1,7 +1,8 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
-shared_examples "a parent indexer" do
+context "when using elasticsearch 5.x and before", :integration => true, :version_less_than_equal_to_5x => true do
+  shared_examples "a type based parent indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
     let(:type) { 10.times.collect { rand(10).to_s }.join("") }
     let(:event_count) { 10000 + rand(500) }
@@ -15,11 +16,10 @@ shared_examples "a parent indexer" do
     before do
       # Add mapping and a parent document
       index_url = "http://#{get_host_port()}/#{index}"
-      ftw = FTW::Agent.new
       mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
-      ftw.put!("#{index_url}", {:body => mapping.to_json, :headers => default_headers})
+      Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
       pdoc = { "foo" => "bar" }
-      ftw.put!("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers})
+      Manticore.put("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers}).call
 
       subject.register
       subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
@@ -29,43 +29,131 @@ shared_examples "a parent indexer" do
     it "ships events" do
       index_url = "http://#{get_host_port()}/#{index}"
 
-      ftw = FTW::Agent.new
-      ftw.post!("#{index_url}/_refresh")
+      Manticore.post("#{index_url}/_refresh").call
 
       # Wait until all events are available.
       Stud::try(10.times) do
         query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
-        data = ""
-        response = ftw.post!("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
-        response.read_body { |chunk| data << chunk }
+        response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
+        data = response.body
         result = LogStash::Json.load(data)
         cur_count = result["count"]
         insist { cur_count } == event_count
       end
     end
-end
+  end
 
-describe "(http protocol) index events with static parent", :integration => true do
-  it_behaves_like 'a parent indexer' do
-    let(:parent) { "test" }
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "parent" => parent
+  describe "(http protocol) index events with static parent" do
+    it_behaves_like 'a type based parent indexer' do
+      let(:parent) { "test" }
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => parent
+        }
       }
-    }
+    end
   end
-end
 
-describe "(http_protocol) index events with fieldref in parent value", :integration => true do
-  it_behaves_like 'a parent indexer' do
-    let(:config) {
-      {
-        "hosts" => get_host_port,
-        "index" => index,
-        "parent" => "%{link_to}"
+  describe "(http_protocol) index events with fieldref in parent value" do
+    it_behaves_like 'a type based parent indexer' do
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => "%{link_to}"
+        }
       }
+    end
+  end
+end
+
+context "when using elasticsearch 6.x and above", :integration => true, :version_greater_than_equal_to_6x => true do
+
+  shared_examples "a join field based parent indexer" do
+    let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+    let(:event_count) { 10000 + rand(500) }
+    let(:parent) { "not_implemented" }
+    let(:config) { "not_implemented" }
+    let(:parent_id) { "test" }
+    let(:join_field) { "join_field" }
+    let(:parent_relation) { "parent_type" }
+    let(:child_relation) { "child_type" }
+    let(:default_headers) {
+      {"Content-Type" => "application/json"}
     }
+    subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+    before do
+      # Add mapping and a parent document
+      index_url = "http://#{get_host_port()}/#{index}"
+      mapping = {
+        "mappings" => {
+          type => {
+            "properties" => {
+              join_field => {
+                "type" => "join",
+                "relations" => { parent_relation => child_relation }
+              }
+            }
+          }
+        }
+      }
+      Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
+      pdoc = { "message" => "ohayo", join_field => parent_relation }
+      Manticore.put("#{index_url}/#{type}/#{parent_id}", {:body => pdoc.to_json, :headers => default_headers}).call
+
+      subject.register
+      subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => parent_id, "message" => "Hello World!", join_field => child_relation) })
+    end
+
+
+    it "ships events" do
+      index_url = "http://#{get_host_port()}/#{index}"
+
+      Manticore.post("#{index_url}/_refresh").call
+
+      # Wait until all events are available.
+      Stud::try(10.times) do
+        query = { "query" => { "has_parent" => { "parent_type" => parent_relation, "query" => { "match_all" => { } } } } }
+        response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
+        data = response.body
+        result = LogStash::Json.load(data)
+        cur_count = result["count"]
+        insist { cur_count } == event_count
+      end
+    end
+  end
+
+  describe "(http protocol) index events with static parent" do
+    it_behaves_like 'a join field based parent indexer' do
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => parent_id,
+          "document_type" => type,
+          "join_field" => join_field,
+          "manage_template" => false
+        }
+      }
+    end
+  end
+
+  describe "(http_protocol) index events with fieldref in parent value" do
+    it_behaves_like 'a join field based parent indexer' do
+      let(:config) {
+        {
+          "hosts" => get_host_port,
+          "index" => index,
+          "parent" => "%{link_to}",
+          "document_type" => type,
+          "join_field" => join_field,
+          "manage_template" => false
+        }
+      }
+    end
   end
 end

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -44,19 +44,44 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
     end
   end
 
+  describe "bad response codes" do
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+
+    it "should raise a bad response code error" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(500)
+      allow(resp).to receive(:body).and_return("a body")
+
+      expect(subject.manticore).to receive(:get).
+        with(uri.to_s + "/", anything).
+        and_return(resp)
+
+      uri_with_path = uri.clone
+      uri_with_path.path = "/"
+
+      expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
+        with(resp.code, uri_with_path, nil, resp.body).and_call_original
+
+      expect do
+        subject.perform_request(uri, :get, "/")
+      end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
+    end
+  end
+
   describe "format_url" do
     let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
     let(:path) { "_bulk" }
     subject { described_class.new(double("logger"), {}) }
 
     it "should add the path argument to the uri's path" do
-      expect(java.net.URI.new(subject.format_url(url, path)).path).to eq("/path/_bulk")
+      expect(subject.format_url(url, path).path).to eq("/path/_bulk")
     end
 
     context "when uri contains query parameters" do
       let(:query_params) { "query=value&key=value2" }
       let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
-      let(:formatted) { java.net.URI.new(subject.format_url(url, path))}
+      let(:formatted) { subject.format_url(url, path)}
 
       it "should retain query_params after format" do
         expect(formatted.query).to eq(query_params)
@@ -73,7 +98,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
     
     context "when the path contains query parameters" do
       let(:path) { "/special_bulk?pathParam=1"}
-      let(:formatted) { java.net.URI.new(subject.format_url(url, path)) }
+      let(:formatted) { subject.format_url(url, path) }
       
       it "should add the path correctly" do
         expect(formatted.path).to eq("#{url.path}special_bulk")
@@ -86,10 +111,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
 
     context "when uri contains credentials" do
       let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
-      let(:formatted) { java.net.URI.new(subject.format_url(url, path)) }
+      let(:formatted) { subject.format_url(url, path) }
 
       it "should remove credentials after format" do
-        expect(formatted.user_info).to be_nil
+        expect(formatted.userinfo).to be_nil
       end
     end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -206,8 +206,10 @@ describe "outputs/elasticsearch" do
         )
       end
       let(:logger) { double("logger").as_null_object }
+      let(:response) { { :errors => [], :items => [] } }
 
       before(:each) do
+
         i = 0
         bulk_param =  [["index", anything, event.to_hash]]
 
@@ -219,7 +221,7 @@ describe "outputs/elasticsearch" do
           if i == 1
             raise error
           end
-        end
+        end.and_return(response)
         eso.multi_receive([event])
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 8.0.1
+  version: 8.1.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2017-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement