logstash-integration-logstash 0.0.4-java → 1.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27a5393272c2cfde9c75782f701b1751dd1d3b35225876fb54c36f13ba0696b6
-  data.tar.gz: d7d9337f2c85dcaddfbc7ef79723e3993dda720fe407b5b189d8bc9ee4e1570a
+  metadata.gz: 46207814a176a2b931e8ce99b90c00f6b5dd7702369743e639d8e98a6aa8d207
+  data.tar.gz: c551712f4de10c93a1220df7d0aac3d4495e49e7a75a8b20f4fb5e57f10e408c
 SHA512:
-  metadata.gz: e4bf91d282d3bfac6a0dae1864db3da98f6ffa02623b9f68eb683d77155d97910b7480442f53a992dda28883a05582e610bc037499b4ed5fcbf51a33b19d65f0
-  data.tar.gz: c6eeb96c776110b415e48d0ce2299f102a3469aaf9673b8870b8e46c02f569c2cf0fd21782a273aa34923673b824730cf128a1569c1e38fdfbc9d831642eb5a9
+  metadata.gz: 20a1f184bd2b45be391f0d8f2e32eb4af5386881ab40eead41328eef107abcc204b36b06fe859006371ae56a80f54217f4c4954b16decd40040e6f53ccf26649
+  data.tar.gz: 692844a4744aab3bd8de0688f0a53cd3cf7adf9234ad0df7c7fe5c6db8b23c7b9eeb3ff82c5a5f5eb09d978442fe9346c03055eaf86c0c9f02d182324474d33d

data/CHANGELOG.md

@@ -1,10 +1,16 @@
+## 1.0.0
+  - Introduces the load balancing mechanism to distribute the requests among the `hosts` [#16](https://github.com/logstash-plugins/logstash-integration-logstash/pull/16)
+
+## 0.0.5
+  - [DOC] Fixes to link formatting [#15](https://github.com/logstash-plugins/logstash-integration-logstash/pull/15)
+
 ## 0.0.4
   - Simplify configuration [#13](https://github.com/logstash-plugins/logstash-integration-logstash/pull/13)
     - Introduce a default `port` of `9800` for both input and output plugins
     - BREAKING: Introduce `hosts` config to output plugin, _replacing_ its separate `host` and `port` configurations
 
 ## 0.0.3
-  - Doc: Minor doc changes and version bump to facilitate adding integration files to doc build [#14](https://github.com/logstash-plugins/logstash-integration-logstash/pull/14)
+  - [DOC] Minor doc changes and version bump to facilitate adding integration files to doc build [#14](https://github.com/logstash-plugins/logstash-integration-logstash/pull/14)
 
 ## 0.0.2
   - Enable data compression [#10](https://github.com/logstash-plugins/logstash-integration-logstash/pull/10)

data/VERSION

@@ -1 +1 @@
-0.0.4
+1.0.0

data/docs/index.asciidoc

@@ -21,15 +21,15 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another:
+The Logstash Integration Plugin provides integrated plugins for sending events from one Logstash to another instance(s):
 
-* {logstash-ref}/plugins-outputs-logstash.html[Logstash Output Plugin]
-* {logstash-ref}/plugins-inputs-logstash.html[Logstash Input Plugin]
+* {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin]
+* {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin]
 
 [id="plugins-{type}s-{plugin}-concepts"]
 ===== High-level concepts
 
-You can configure a `logstash` output to send events to a `logstash` input in another pipeline that is running in a different process or on a different host.
+You can configure a `logstash` output to send events to one or more `logstash` inputs, which are each in another pipeline that is running in different processes or on a different host.
 
 To do so, you should first configure the downstream pipeline with a `logstash` input plugin, bound to an available port so that it can listen for inbound connections.
 Security is enabled by default, so you will need to either provide identity material or disable SSL.
@@ -53,13 +53,11 @@ input {
 Once the downstream pipeline is configured and running, you may send events from any number of upstream pipelines by adding a `logstash` output plugin that points to the downstream input.
 You may need to configure SSL to trust the certificates presented by the downstream input plugin.
 
-NOTE: Single host endpoint is supported for `hosts`. Multi-host support is coming soon.
-
 [source]
 ----
 output {
   logstash {
-    hosts => "10.0.0.123:9800"
+    hosts => ["10.0.0.123:9800", "10.0.0.125:9801"]
 
     # SSL TRUST <1>
     ssl_truststore_path => "/path/to/truststore.p12"
@@ -69,4 +67,9 @@ output {
 ----
 <1> Unless SSL is disabled or the downstream input is expected to present certificates signed by globally-trusted authorities, you will likely need to provide a source-of-trust.
 
+[id="plugins-{type}s-{plugin}-load-balancing"]
+==== Load Balancing
+
+When a `logstash` output is configured to send to multiple `hosts`, it distributes events in batches to _all_ of those downstream hosts fairly, favoring those without recent errors. This increases the likelihood of each batch being routed to a downstream that is up and has capacity to receive events.
+
 :no_codec!:

data/docs/input-logstash.asciidoc

@@ -22,7 +22,7 @@ include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
-Listen for events that are sent by a <<plugins-outputs-logstash>> in a pipeline that may be in another process or on another host.
+Listen for events that are sent by a {logstash-ref}/plugins-outputs-logstash.html[Logstash output plugin] in a pipeline that may be in another process or on another host.
 The upstream output must have a TCP route to the port (defaults to 9800) on an interface that this plugin is bound to.
 
 NOTE: Sending events to this input by _any_ means other than `plugins-outputs-logstash` is neither advised nor supported.
@@ -91,7 +91,7 @@ NOTE: Client-certificate verification does _not_ verify identity claims on the p
 ===== Security: Credentials
 
 You can also configure this plugin to require a specific username/password be provided by configuring <<plugins-{type}s-{plugin}-username>> and <<plugins-{type}s-{plugin}-password>>.
-Doing so requires connecting <<plugins-outputs-logstash>> clients to provide matching <<plugins-outputs-{plugin}-username>> and <<plugins-outputs-{plugin}-password>>.
+Doing so requires connecting `logstash-output` plugin clients to provide matching `username` and `password`. 
 
 NOTE: when SSL is disabled, data and credentials will be received in clear-text.
 
@@ -240,7 +240,7 @@ A password or passphrase of the <<plugins-{type}s-{plugin}-ssl_key>>.
 * There is no default value for this setting.
 
 Username for password-based authentication.
-When this input plugin is configured with a `username`, it also requires a <<plugins-{type}s-{plugin}-password>>, and any upstream <<plugins-outputs-logstash>> must also be configured with a matching `username`/`password` pair.
+When this input plugin is configured with a `username`, it also requires a `password`, and any upstream `logstash-output` plugin must also be configured with a matching `username`/`password` pair.
 
 NOTE: when SSL is disabled, credentials will be transmitted in clear-text.
 

data/docs/output-logstash.asciidoc

@@ -22,10 +22,10 @@ include::{include_path}/plugin_header-integration.asciidoc[]
 
 ==== Description
 
-Send events to a <<plugins-inputs-logstash>> in a pipeline that may be in another process or on another host.
+Send events to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] in a pipeline that may be in another process or on another host.
 You must have a TCP route to the port (defaults to 9800) on an interface that the downstream input is bound to.
 
-NOTE: Sending events to _any_ destination other than <<plugins-inputs-logstash>> is neither advised nor supported.
+NOTE: Sending events to _any_ destination other than a `logstash-input` plugin is neither advised nor supported.
       We will maintain cross-compatibility with any two supported versions of output/input pair and reserve the right to change details such as protocol and encoding.
 
 [id="plugins-{type}s-{plugin}-minimum-config"]
@@ -63,7 +63,7 @@ output {
 [id="plugins-{type}s-{plugin}-config-connecting"]
 ===== Configuration Concepts
 
-This output plugin needs to be configured to connect to a <<plugins-inputs-logstash>> by specifying its <<plugins-{type}s-{plugin}-hosts>>.
+Configure this output plugin to connect to a {logstash-ref}/plugins-inputs-logstash.html[Logstash input plugin] by specifying its `hosts`.
 Depending on the downstream plugin's configuration, you may need to also configure the target port, SSL, and/or credentials.
 
 [id="plugins-{type}s-{plugin}-config-ssl-trust"]
@@ -90,7 +90,7 @@ If the downstream input plugin is configured to request or require client authen
 [id="plugins-{type}s-{plugin}-config-credentials"]
 ===== Security: Credentials
 
-If the downstream <<plugins-inputs-logstash>> is configured to require <<plugins-inputs-logstash-username>> and <<plugins-inputs-logstash-password>>,
+If the downstream `logstash-input` plugin is configured to require `username` and `password`,
 you will need to configure this output with a matching <<plugins-{type}s-{plugin}-username>> and <<plugins-{type}s-{plugin}-password>>.
 
 NOTE: when SSL is disabled, data and credentials will be transmitted in clear-text.
@@ -103,7 +103,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting                            |Input type        |Required
-| <<plugins-{type}s-{plugin}-hosts>> |<<string,string>> |Yes
+| <<plugins-{type}s-{plugin}-hosts>> |list of <<string,string>> |Yes
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-ssl_enabled>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_certificate>> | <<path,path>>|No
@@ -125,27 +125,27 @@ output plugins.
 [id="plugins-{type}s-{plugin}-hosts"]
 ===== `hosts`
 
-* Value type is <<string,string>>
+* Value type is list of <<string,string>>
 * There is no default value for this setting.
 * Constraints:
 ** When using IPv6, IP address must be in an enclosed in brackets.
 ** When a port is not provided, the default `9800` is used.
 
-A downstream input {ls} host or IP address to connect.
-
-NOTE: Single host endpoint is supported for `hosts`. Multi-host support is coming soon.
+The addresses of one or more downstream `input`s to connect to.
 
 Host can be any of IPv4, IPv6 (in enclosed bracket) or host name, examples:
 
 * `"127.0.0.1"`
 * `"127.0.0.1:9801"`
 * `"ds.example.com"`
-* `"ds.example:9802"`
+* `"ds.example.com:9802"`
 * `"[::1]"`
 * `"[::1]:9803"`
 * `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"`
 * `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:9804"`
 
+Plugin balances incoming load among the `hosts`. For more information, visit {logstash-ref}/plugins-integrations-logstash.html[Logstash integration plugin] _Load Balancing_ section.
+
 When connecting, communication to downstream input {ls} is secured with SSL unless configured otherwise.
 
 [WARNING]

data/lib/logstash/outputs/logstash.rb

@@ -1,58 +1,59 @@
 # encoding: utf-8
 
-require 'logstash/outputs/base'
-require 'logstash/namespace'
 
-require "logstash/plugin_mixins/plugin_factory_support"
+require "logstash/outputs/base"
+require "logstash/namespace"
+
+require 'logstash/plugin_mixins/normalize_config_support'
+require "logstash/plugin_mixins/http_client"
 require "logstash/plugin_mixins/validator_support/required_host_optional_port_validation_adapter"
+require "zlib"
+
+require "stud/interval" # Stud::stoppable_sleep
 
 class LogStash::Outputs::Logstash < LogStash::Outputs::Base
   extend LogStash::PluginMixins::ValidatorSupport::RequiredHostOptionalPortValidationAdapter
 
-  include LogStash::PluginMixins::PluginFactorySupport
+  include LogStash::PluginMixins::HttpClient[:with_deprecated => false]
+  include LogStash::PluginMixins::NormalizeConfigSupport
+
+  require "logstash/utils/load_balancer"
 
   config_name "logstash"
 
   # Sets the host of the downstream Logstash instance.
   # Host can be any of IPv4, IPv6 (requires to be in enclosed bracket) or host name, the forms:
-  #     `"127.0.0.1"`
-  #     `"127.0.0.1:9800"`
-  #     `"foo-bar.com"`
-  #     `"foo-bar.com:9800"`
-  #     `"[::1]"`
-  #     `"[::1]:9000"`
+  #     `"127.0.0.1"` or `["127.0.0.1"]` if single host with default port
+  #     `"127.0.0.1:9801"` or `["127.0.0.1:9801"]` if single host with custom port
+  #     `["foo-bar.com", "foo-bar.com:9800"]`
+  #     `["[::1]", "[::1]:9000"]`
   #     `"[2001:0db8:85a3:0000:0000:8a2e:0370:7334]"`
   #
-  # NOTE: `hosts` naming is intentional and multi-host support is planned.
-  #
-  config :hosts, :validate => :required_host_optional_port, :required => true
-
-  # optional username/password credentials
-  config :username, :validate => :string,   :required => false
-  config :password, :validate => :password, :required => false
+  config :hosts, :validate => :required_host_optional_port, :list => true, :required => true
 
-  config :ssl_enabled,                 :validate => :boolean, :default => true
+  config :username, :validate => :string, :required => false
 
-  # SSL:IDENTITY:SOURCE cert/key pair
-  config :ssl_certificate,             :validate => :path
-  config :ssl_key,                     :validate => :path
+  config :ssl_enabled, :validate => :boolean, :default => true
 
-  # SSL:IDENTITY:SOURCE keystore
-  config :ssl_keystore_path,           :validate => :path
-  config :ssl_keystore_password,       :validate => :password
+  config :user, :validate => :string, :deprecated => "Use `username` instead.", :required => false
 
-  # SSL:TRUST:CONFIG
-  config :ssl_verification_mode,       :validate => %w(full none), :default => 'full'
-
-  # SSL:TRUST:SOURCE ca file
-  config :ssl_certificate_authorities, :validate => :path, :list => true
-
-  # SSL:TRUST:SOURCE truststore
-  config :ssl_truststore_path,         :validate => :path
-  config :ssl_truststore_password,     :validate => :password
+  DEFAULT_PORT = 9800.freeze
 
-  # SSL:TUNING
-  config :ssl_supported_protocols, :validate => :string, :list => true
+  RETRIABLE_CODES = [429, 500, 502, 503, 504]
+  RETRYABLE_MANTICORE_EXCEPTIONS = [
+    ::Manticore::Timeout,
+    ::Manticore::SocketException,
+    ::Manticore::ClientProtocolException,
+    ::Manticore::ResolutionFailure,
+    ::Manticore::SocketTimeout
+  ]
+  RETRYABLE_EXCEPTION_PATTERN = Regexp.union([
+    /Connection reset by peer/i,
+    /Read Timed out/i,
+  ])
+
+  # @api private
+  attr_reader :http_client
 
   def initialize(*a)
     super
@@ -61,129 +62,252 @@ class LogStash::Outputs::Logstash < LogStash::Outputs::Base
       fail LogStash::ConfigurationError, 'The `logstash` output does not have an externally-configurable `codec`'
     end
 
-    if @ssl_certificate_authorities && @ssl_certificate_authorities.size > 1
-      fail LogStash::ConfigurationError, 'The `logstash` output supports at most one `ssl_certificate_authorities` path'
-    end
+    @headers = {
+      "Content-Type" => "application/x-ndjson".freeze,
+      "Content-Encoding" => "gzip".freeze
+    }.freeze
 
-    logger.debug("initializing inner HTTP output plugin")
-    @internal_http = plugin_factory.output('http').new(inner_http_output_options)
-    logger.debug("inner HTTP output plugin has been initialized")
+    logger.debug("`logstash` output plugin has been initialized")
   end
 
   def register
-    logger.debug("registering inner HTTP output plugin")
-    @internal_http.register
-    logger.debug("inner HTTP output plugin has been registered")
+    logger.debug("Registering `logstash` output plugin")
+
+    @username = normalize_config(:username) do |normalize|
+      normalize.with_deprecated_alias(:user)
+    end
+    # remove after deprecating user in the http-mixin
+    @user = @username ? @username.freeze : @user
+
+    validate_auth_settings!
+
+    if @ssl_enabled == false
+      rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
+      fail(LogStash::ConfigurationError, "Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
+    end
+
+    validate_ssl_identity_options!
+    validate_ssl_trust_options!
+
+    # if we don't initialize now, we get runtime error when sending events if there are issues with configs
+    @http_client = client
+    fail(LogStash::ConfigurationError, "`hosts` must not be empty") if @hosts.empty?
+
+    @load_balancer = LoadBalancer.new(normalize_host_uris)
+
+    logger.debug("`logstash` output plugin has been registered")
+  end
+
+  def validate_auth_settings!
+    if @username
+      fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided') if @password.nil?
+      logger.warn("Transmitting credentials over non-secured connection") if @ssl_enabled == false
+    elsif @password
+      fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
+    end
+  end
+
+  def validate_ssl_identity_options!
+    if @ssl_certificate && @ssl_keystore_path
+      fail(LogStash::ConfigurationError, "SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both")
+    elsif @ssl_certificate
+      fail(LogStash::ConfigurationError, "`ssl_key` is REQUIRED when `ssl_certificate` is provided") if @ssl_key.nil?
+    elsif @ssl_key
+      fail(LogStash::ConfigurationError, "`ssl_key` is not allowed unless `ssl_certificate` is configured")
+    elsif @ssl_keystore_path
+      fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided") if @ssl_keystore_password.nil?
+    elsif @ssl_keystore_password
+      fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
+    else
+      # acceptable
+    end
+  end
+
+  def validate_ssl_trust_options!
+    if @ssl_certificate_authorities&.any? && @ssl_truststore_path
+      fail(LogStash::ConfigurationError, "SSL trust can be configured with EITHER `ssl_certificate_authorities` OR `ssl_truststore_*`, but not both")
+    elsif @ssl_certificate_authorities&.any?
+      fail(LogStash::ConfigurationError, "SSL Certificate Authorities cannot be configured when `ssl_verification_mode => none`") if @ssl_verification_mode == 'none'
+    elsif @ssl_truststore_path
+      fail(LogStash::ConfigurationError, "SSL Truststore cannot be configured when `ssl_verification_mode => none`") if @ssl_verification_mode == 'none'
+      fail(LogStash::ConfigurationError, "`ssl_truststore_password` is REQUIRED when `ssl_truststore_path` is provided") if @ssl_truststore_password.nil?
+    elsif @ssl_truststore_password
+      fail(LogStash::ConfigurationError, "`ssl_truststore_password` not allowed unless `ssl_truststore_path` is configured")
+    end
   end
 
   def multi_receive(events)
     return if events.empty?
-    logger.trace("proxying #{events.size} events to inner HTTP plugin")
-    @internal_http.multi_receive(events)
-  rescue => e
-    logger.error("inner HTTP plugin has had an unrecoverable exception: #{e.message} at #{e.backtrace.first}")
-    raise
+
+    send_events(events)
   end
 
   def stop
-    logger.debug("stopping inner HTTP output plugin")
-    @internal_http.stop
-    logger.debug('inner HTTP output plugin has been stopped')
+    logger.debug("`logstash` output plugin has been stopped")
   end
 
   def close
-    logger.debug("closing inner HTTP output plugin")
-    @internal_http.close
-    logger.debug('inner HTTP output plugin has been closed')
+    logger.debug("Closing `logstash` output plugin")
+    http_client.close
+    logger.debug("`logstash` output plugin has been closed")
   end
 
-  DEFAULT_PORT = 9800.freeze
+  private
 
-  def inner_http_output_options
-    @_inner_http_output_options ||= begin
-      http_options = {
-        'url'                  => construct_host_uri.to_s,
-        'http_method'          => 'post',
-        'retry_non_idempotent' => 'true',
-
-        # non-configurable codec
-        'content_type' => 'application/x-ndjson',
-        'format'       => 'json_batch',
-
-        'http_compression' => true,
-      }
-
-      if @username
-        http_options['user'] = @username
-        http_options['password'] = @password || fail(LogStash::ConfigurationError, '`password` is REQUIRED when `username` is provided')
-        logger.warn("transmitting credentials over non-secured connection") if @ssl_enabled == false
-      elsif @password
-        fail(LogStash::ConfigurationError, '`password` not allowed unless `username` is configured')
-      end
+  def normalize_host_uris
+    @_normalized_host_uris ||= begin
+      scheme = @ssl_enabled ? 'https' : 'http'
+      @hosts.map do |destination| # Struct(:host,:port)
+        URI::Generic.build(:scheme => scheme,
+                           :host   => destination.host,
+                           :port   => destination.port || DEFAULT_PORT)
+      end.map(&:to_s).map(&:freeze)
+    end
+  end
 
-      if @ssl_enabled == false
-        rejected_ssl_settings = @original_params.keys.select { |k| k.start_with?('ssl_') } - %w(ssl_enabled)
-        fail(LogStash::ConfigurationError, "Explicit SSL-related settings not supported because `ssl_enabled => false`: #{rejected_ssl_settings}") if rejected_ssl_settings.any?
-      else
-        http_options['ssl_supported_protocols'] = @ssl_supported_protocols if @original_params.include?('ssl_supported_protocols')
+  def send_events(events)
+    body = LogStash::Json.dump(events.map(&:to_hash))
+    compressed_body = gzip(body)
 
-        http_options.merge!(ssl_identity_options)
-        http_options.merge!(ssl_trust_options)
-      end
+    next_backoff = 0.1
+    max_backoff = 30
+
+    loop do
+      next_action = transmit(body, compressed_body)
+      break unless next_action == :retry
 
-      http_options
+      Stud.stoppable_sleep(next_backoff) { pipeline_shutdown_requested? }
+      next_backoff = [next_backoff*2, max_backoff].min
+
+      if pipeline_shutdown_requested?
+        logger.warn "Aborting the batch due to shutdown request"
+        abort_batch_if_available!
+        break # legacy abort (lossy)
+      end
     end
+  rescue => e
+    # This should never happen unless there's a flat out bug in the code
+    logger.error("Error occurred while sending events",
+                 :class => e.class.name,
+                 :message => e.message,
+                 :backtrace => e.backtrace)
+    raise e
   end
 
-  def construct_host_uri
-    scheme = @ssl_enabled ? 'https'.freeze : 'http'.freeze
-    host_port_pair = @hosts # Struct(:host, :port)
-    uri = LogStash::Util::SafeURI.new(host_port_pair[:host])
-    uri.port = host_port_pair[:port].nil? ? DEFAULT_PORT : host_port_pair[:port]
-    uri.update(:scheme, scheme)
-    uri.freeze
+  # The exceptions we decide (retriable or abort) to let the Load Balancer know
+  TransmitException = Class.new(RuntimeError)
+  private_constant :TransmitException
+
+  RetriableTransmitException = Class.new(TransmitException)
+  private_constant :RetriableTransmitException
+
+  TerminalTransmitException = Class.new(TransmitException)
+  private_constant :TerminalTransmitException
+
+  ##
+  # @param body [String]
+  # @param compressed_body [String]
+  # @return [:done, :abort, :retry]
+  def transmit(body, compressed_body)
+    @load_balancer.select do |selected_host_uri|
+      response = begin
+                   http_client.post(selected_host_uri, :body => compressed_body, :headers => @headers).call
+                 rescue => exception
+                   retryable_exception = retryable_exception?(exception)
+                   log_exception(selected_host_uri, exception, body, retryable_exception)
+
+                   # raise exception to mar the host error
+                   raise retryable_exception ? RetriableTransmitException : TerminalTransmitException
+                 end
+
+      return :done if response_success?(response.code)
+
+      retryable_response = retryable_response?(response.code)
+      log_response(selected_host_uri, response, body, retryable_response)
+
+      # raise exception to mar the host error
+      raise retryable_response ? RetriableTransmitException : TerminalTransmitException
+    end
+  rescue RetriableTransmitException => exception
+    return :retry
+  rescue TerminalTransmitException => exception
+    return :abort
   end
 
-  def ssl_identity_options
-    if @ssl_certificate && @ssl_keystore_path
-      fail(LogStash::ConfigurationError, 'SSL identity can be configured with EITHER `ssl_certificate` OR `ssl_keystore_*`, but not both')
-    elsif @ssl_certificate
-      return {
-        'ssl_certificate' => @ssl_certificate,
-        'ssl_key'  => @ssl_key || fail(LogStash::ConfigurationError, "`ssl_key` is REQUIRED when `ssl_certificate` is provided"),
-      }
-    elsif @ssl_key
-      fail(LogStash::ConfigurationError, '`ssl_key` is not allowed unless `ssl_certificate` is configured')
-    elsif @ssl_keystore_path
-      return {
-        'ssl_keystore_path' => @ssl_keystore_path,
-        'ssl_keystore_password' => @ssl_keystore_password || fail(LogStash::ConfigurationError, "`ssl_keystore_password` is REQUIRED when `ssl_keystore_path` is provided"),
-      }
-    elsif @ssl_keystore_password
-      fail(LogStash::ConfigurationError, "`ssl_keystore_password` is not allowed unless `ssl_keystore_path` is configured")
+  def log_response(uri, response, body, retriable)
+    response_code = response.code
+    if retriable
+      if response_code == 429
+        logger.debug("Encountered a retriable 429 response")
+      else
+        logger.warn("Encountered a retryable error in `logstash` output", :code => response_code, :body => response.body)
+      end
     else
-      return {}
+      logger.error("Encountered error",
+                   :response_code => response_code,
+                   :host => uri,
+                   :body => body
+      )
+    end
+  end
+
+  def log_exception(uri, exception, body, retriable)
+    log_entry = { :host => uri, :message => exception.message, :class => exception.class, :retry => retriable }
+    if logger.debug?
+      # backtraces are big
+      log_entry[:backtrace] = exception.backtrace
+      # body can be big and may have sensitive data
+      log_entry[:body] = body
     end
+    logger.error("Could not send data to host", log_entry)
   end
 
-  def ssl_trust_options
-    {
-      'ssl_verification_mode' => @ssl_verification_mode,
-    }.tap do |trust_options|
-      if @ssl_certificate_authorities&.any? && @ssl_truststore_path
-        fail(LogStash::ConfigurationError, 'SSL trust can be configured with EITHER `ssl_certificate_authorities` OR `ssl_truststore_*`, but not both')
-      elsif @ssl_certificate_authorities&.any?
-        fail(LogStash::ConfigurationError, 'SSL Certificate Authorities cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
+  def gzip(data)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz)
+    z.write(data)
+    z.close
+    gz.string
+  end
 
-        trust_options['ssl_certificate_authorities'] = @ssl_certificate_authorities.first
-      elsif @ssl_truststore_path
-        fail(LogStash::ConfigurationError, 'SSL Truststore cannot be configured when `ssl_verification_mode => none`') if @ssl_verification_mode == 'none'
+  def response_success?(response_code)
+    response_code >= 200 && response_code <= 299
+  end
 
-        trust_options['ssl_truststore_path'] = @ssl_truststore_path
-        trust_options['ssl_truststore_password'] = @ssl_truststore_password || fail(LogStash::ConfigurationError, '`ssl_truststore_password` is REQUIRED when `ssl_truststore_path` is provided')
-      elsif @ssl_truststore_password
-        fail(LogStash::ConfigurationError, '`ssl_truststore_password` not allowed unless `ssl_truststore_path` is configured')
-      end
+  def retryable_exception?(exception)
+    retryable_manticore_exception?(exception) || retryable_unknown_exception?(exception)
+  end
+
+  def retryable_manticore_exception?(exception)
+    RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me)}
+  end
+
+  def retryable_unknown_exception?(exception)
+    exception.is_a?(::Manticore::UnknownException) &&
+      RETRYABLE_EXCEPTION_PATTERN.match?(exception.message)
+  end
+
+  def retryable_response?(response_code)
+    RETRIABLE_CODES.include?(response_code)
+  end
+
+  # Emulate `pipeline_shutdown_requested?` when running on older Logstash
+  unless ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.1.0')
+    def pipeline_shutdown_requested?
+      execution_context&.pipeline&.shutdown_requested?
+    end
+  end
+
+  # When running on Logstash that can abort batches,
+  # raise the required exception, do nothing otherwise.
+  if ::Gem::Version.create(LOGSTASH_VERSION) >= ::Gem::Version.create('8.8.0')
+    def abort_batch_if_available!
+      raise org.logstash.execution.AbortedBatchException.new
+    end
+  else
+    def abort_batch_if_available!
+      nil
     end
   end
 end