logstash-output-amazon_es 6.4.1-java → 7.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17eb123cc2139d8477a04a8a3d809d43b49010f9fe8582933193ca16ebe6b44f
-  data.tar.gz: 420352d6cd91730c4fd8d61248336d9cda161a676518768d6a8c958e522b28eb
+  metadata.gz: 827216d23ab630ee15e9ab6fd7324ef2dd506ff3ca7296ada51afa09e78b08e3
+  data.tar.gz: 67471b7f10a922559cd3247a370f80b5989c30947bd3981ea2233c1b0f310a18
 SHA512:
-  metadata.gz: 6d9ac4617b99e7703375d34cbaf45ad4610107eafacbb21287fc68eb83a37e8d7ebe139b12fb74b6204d80aa32e37b1bdf3ec527762cd067c377169c0024e985
-  data.tar.gz: 78bf8133942d515e8d268eed0609da9202b4bdd1789ed4ed9c9fe40ca01df62227249d104b50fe2721e8deaea3f63ffa22673627cd65a6873c9de3ca9b1f33ca
+  metadata.gz: 7ce4a46376dce1d4f0cd684086b28d94c30fd1fb841e59562713b2c674b3ed01df89e6257a78612df7cad1b88569b771e2e513581981366994563fecd7d23edc
+  data.tar.gz: b7c513e943a05a6e78cf602d66cf6139d42a36881acd23b7d7fbebb1319d61c9ceff7753b2c1303f3c29b0afcf621f80d741e013bf8e79a35cdc2561bbe76694

data/Gemfile

@@ -1,3 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 source 'https://rubygems.org'
 
 gemspec

data/NOTICE.TXT

@@ -1,14 +1,14 @@
 Logstash-output-amazon_es Plugin
-Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+This product includes software from the Apache-2.0 licensed Elasticsearch project.
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expressed or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+It requires the following notice:
+
+  Elasticsearch
+  Copyright 2012-2015 Elasticsearch
+  Copyright 2020 Elastic and contributors
+
+  This product includes software developed by The Apache Software
+  Foundation (http://www.apache.org/).

data/README.md

@@ -1,6 +1,9 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elastic/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash) which outputs
+to [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/) 
+(successor to Amazon Elasticsearch Service) using
+[SigV4 signing](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html).
 
 ## License
 
@@ -8,16 +11,43 @@ This library is licensed under Apache License 2.0.
 
 ## Compatibility
 
-The following table shows the versions of logstash and logstash-output-amazon_es Plugin was built with.
+The following table shows the versions of logstash and logstash-output-amazon_es plugin was built with.
 
-|  logstash-output-amazon_es | Logstash |
-| ------------- | ------------- |
-| 6.0.0  | <6.0.0  |
-| 6.4.0  | >6.0.0  |
+| logstash-output-amazon_es | Logstash |
+|---------------------------|----------|
+| 6.0.0                     | < 6.0.0  |
+| 6.4.2                     | >= 6.0.0 |
+| 7.0.1                     | >= 7.0.0 |
 
-## Configuration for Amazon Elasticsearch Output Plugin
 
-To run the Logstash Output Amazon Elasticsearch plugin, simply add a configuration following the below documentation.
+Also, logstash-output-amazon_es plugin versions 6.4.0 and newer are tested to be compatible with Elasticsearch 6.5 and greater. 
+
+|  logstash-output-amazon_es | Elasticsearch |
+| ------------- |----------|
+| 6.4.0+  | 6.5+     |
+
+
+## Installation
+
+To install the latest version, use the normal Logstash plugin script.
+
+```sh
+bin/logstash-plugin install logstash-output-amazon_es
+```
+
+If you want to use old version of logstash-output-amazon_es, you can use the `--version`
+flag to specify the version. For example:
+
+```sh
+bin/logstash-plugin install --version 6.4.2 logstash-output-amazon_es
+```
+
+
+
+
+## Configuration for Amazon Elasticsearch Service Output Plugin
+
+To run the Logstash Output Amazon Elasticsearch Service plugin, simply add a configuration following the below documentation.
 
 An example configuration:
 
@@ -46,30 +76,48 @@ output {
   * aws_access_key_id, :validate => :string - optional AWS access key
   * aws_secret_access_key, :validate => :string - optional AWS secret key
 
-	 The credential resolution logic can be described as follows:
+   The credential resolution logic can be described as follows:
 
-	 - User passed `aws_access_key_id` and `aws_secret_access_key` in `amazon_es` configuration
-	 - Environment variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), or `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` (only recognized by Java SDK)
-	 - Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI
-	 - Instance profile credentials delivered through the Amazon EC2 metadata service
+   - User passed `aws_access_key_id` and `aws_secret_access_key` in `amazon_es` configuration
+   - Environment variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET), or `AWS_ACCESS_KEY` and `AWS_SECRET_KEY` (only recognized by Java SDK)
+   - Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI
+   - Instance profile credentials delivered through the Amazon EC2 metadata service
 
 - template (path) - You can set the path to your own template here, if you so desire. If not set, the included template will be used.
 - template_name (string, default => "logstash") - defines how the template is named inside Elasticsearch
 - port (string, default 443) - Amazon Elasticsearch Service listens on port 443 for HTTPS (default) and port 80 for HTTP. Tweak this value for a custom proxy.
 - protocol (string, default https) - The protocol used to connect to the Amazon Elasticsearch Service
+- max_bulk_bytes - The max size for a bulk request in bytes. Default is 20MB. It is recommended not to change this value unless needed. For guidance on changing this value, please consult the table for network limits for your instance type: https://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html#network-limits
 
 After 6.4.0, users can't set batch size in this output plugin config. However, users can still set batch size in logstash.yml file.
+
+### Advanced Optional Parameters
+
+Starting logstash-output-amazon_es v7.1.0, we have introduced the following optional parameters to resolve specific use cases:
+
+- service_name (string, default => "es") - Users can define any service name to which the plugin will send a SigV4 signed request
+- skip_healthcheck (boolean, default => false) - Boolean to skip healthcheck API and set the major ES version to 7
+- skip_template_installation (boolean, default => false) -  Boolean to allow users to skip installing templates in usecases that don't require them
+
 ## Developing
 
-### 1. Plugin Development and Testing
+### 1. Prerequisites
+To get started, you can install JRuby with the Bundler gem using [RVM](https://rvm.io/rvm/install)
+
+rvm install jruby
+
+### 2. Plugin Development and Testing
 
 #### Code
 
-1. To get started, you'll need JRuby with the Bundler gem installed.
+1. Verify JRuby is already installed
+
+   ```sh
+   jruby -v
+   ```
 
-2. Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. [Example plugins](https://github.com/logstash-plugins?query=example) exist.
 
-3. Install dependencies:
+2. Install dependencies:
 
    ```sh
    bundle install
@@ -89,14 +137,14 @@ After 6.4.0, users can't set batch size in this output plugin config. However, u
    bundle exec rspec
    ```
 
-### 2. Running your unpublished plugin in Logstash
+### 3. Running your unpublished plugin in Logstash
 
-#### 2.1 Run in a local Logstash clone
+#### 3.1 Run in a local Logstash clone
 
 1. Edit Logstash `Gemfile` and add the local plugin path, for example:
 
    ```ruby
-   gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+   gem "logstash-output-amazon_es", :path => "/your/local/logstash-output-amazon_es"
    ```
 
 2. Install the plugin:
@@ -112,12 +160,12 @@ After 6.4.0, users can't set batch size in this output plugin config. However, u
 3. Run Logstash with your plugin:
 
    ```sh
-   bin/logstash -e 'filter {awesome {}}'
+   bin/logstash -e 'output {amazon_es {}}'
    ```
 
 At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply re-run Logstash.
 
-#### 2.2 Run in an installed Logstash
+#### 3.2 Run in an installed Logstash
 
 Before build your `Gemfile`, please make sure use JRuby. Here is how you can know your local Ruby version:
 
@@ -128,37 +176,25 @@ rvm list
 Please make sure you current using JRuby. Here is how you can change to JRuby
 
 ```sh
-rvm jruby
+rvm jruby-9.2.5.0
 ```
 
-You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory. You can also build the gem and install it using:
+You can use the same **3.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory. You can also build the gem and install it using:
 
 1. Build your plugin gem:
 
    ```sh
-   gem build logstash-filter-awesome.gemspec
+   gem build logstash-output-amazon_es.gemspec
    ```
 
-2. Install the plugin from the Logstash home:
+2. Install the plugin from the Logstash home. Please be sure to check the version number against the actual Gem file. Run:
 
    ```sh
-   # Logstash 2.3 and higher
-   bin/logstash-plugin install --no-verify
-
-   # Prior to Logstash 2.3
-   bin/plugin install --no-verify
+   bin/logstash-plugin install /your/local/logstash-output-amazon_es/logstash-output-amazon_es-7.0.1-java.gem
    ```
 
 3. Start Logstash and test the plugin.
 
-## Old version support
-
-If you want to use old version of logstash-output-amazon_es, you can install with this:  
-```sh
-bin/logstash-plugin install logstash-output-amazon_es -v 2.0.0
-```
-
-
 
 ## Contributing
 

data/lib/logstash/outputs/amazon_es/common.rb

@@ -1,8 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/outputs/amazon_es/template_manager"
 
-module LogStash; module Outputs; class ElasticSearch;
+module LogStash; module Outputs; class AmazonElasticSearch;
   module Common
-    attr_reader :client, :hosts
+    attr_reader :client, :hosts, :skip_template_installation
 
     # These codes apply to documents, not at the request level
     DOC_DLQ_CODES = [400, 404]
@@ -115,8 +119,12 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def install_template
-      TemplateManager.install_template(self)
-      @template_installed.make_true
+      if skip_template_installation == false
+       TemplateManager.install_template(self)
+       @template_installed.make_true
+      elsif skip_template_installation == true
+        @template_installed.make_true
+      end
     end
 
     def check_action_validity
@@ -274,7 +282,7 @@ module LogStash; module Outputs; class ElasticSearch;
         es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
         response = @client.bulk(es_actions)
         response
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
+      rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::HostUnreachableError => e
         # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
         # and let the user sort it out from there
         @logger.error(
@@ -290,7 +298,7 @@ module LogStash; module Outputs; class ElasticSearch;
         sleep_interval = sleep_for_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
         retry unless @stopping.true?
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
+      rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
         @logger.error(
           "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
           :error_message => e.message,
@@ -301,7 +309,7 @@ module LogStash; module Outputs; class ElasticSearch;
         sleep_interval = next_sleep_interval(sleep_interval)
         @bulk_request_metrics.increment(:failures)
         retry unless @stopping.true?
-      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+      rescue ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
         log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
         log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose

data/lib/logstash/outputs/amazon_es/common_configs.rb

@@ -1,6 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978
 
-module LogStash; module Outputs; class ElasticSearch
+module LogStash; module Outputs; class AmazonElasticSearch
   module CommonConfigs
     def self.included(mod)
       # The index to write events to. This can be dynamic using the `%{foo}` syntax.

data/lib/logstash/outputs/amazon_es/elasticsearch-template-es7x.json

@@ -1,44 +1,42 @@
 {
   "template" : "logstash-*",
-  "version" : 60001,
+  "version" : 60002,
   "settings" : {
     "index.refresh_interval" : "5s",
     "number_of_shards": 1
   },
   "mappings" : {
-    "_doc" : {
-      "dynamic_templates" : [ {
-        "message_field" : {
-          "path_match" : "message",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text",
-            "norms" : false
-          }
+    "dynamic_templates" : [ {
+      "message_field" : {
+        "path_match" : "message",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text",
+          "norms" : false
         }
-      }, {
-        "string_fields" : {
-          "match" : "*",
-          "match_mapping_type" : "string",
-          "mapping" : {
-            "type" : "text", "norms" : false,
-            "fields" : {
-              "keyword" : { "type": "keyword", "ignore_above": 256 }
-            }
+      }
+    }, {
+      "string_fields" : {
+        "match" : "*",
+        "match_mapping_type" : "string",
+        "mapping" : {
+          "type" : "text", "norms" : false,
+          "fields" : {
+            "keyword" : { "type": "keyword", "ignore_above": 256 }
           }
         }
-      } ],
-      "properties" : {
-        "@timestamp": { "type": "date"},
-        "@version": { "type": "keyword"},
-        "geoip"  : {
-          "dynamic": true,
-          "properties" : {
-            "ip": { "type": "ip" },
-            "location" : { "type" : "geo_point" },
-            "latitude" : { "type" : "half_float" },
-            "longitude" : { "type" : "half_float" }
-          }
+      }
+    } ],
+    "properties" : {
+      "@timestamp": { "type": "date"},
+      "@version": { "type": "keyword"},
+      "geoip"  : {
+        "dynamic": true,
+        "properties" : {
+          "ip": { "type": "ip" },
+          "location" : { "type" : "geo_point" },
+          "latitude" : { "type" : "half_float" },
+          "longitude" : { "type" : "half_float" }
         }
       }
     }

data/lib/logstash/outputs/amazon_es/http_client/manticore_adapter.rb

@@ -1,9 +1,13 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require 'manticore'
 require 'cgi'
 require 'aws-sdk-core'
 require 'uri'
 
-module LogStash; module Outputs; class ElasticSearch; class HttpClient;
+module LogStash; module Outputs; class AmazonElasticSearch; class HttpClient;
   DEFAULT_HEADERS = { "content-type" => "application/json" }
 
   CredentialConfig = Struct.new(
@@ -33,12 +37,16 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @port =  options[:port] || 9200
       @protocol =  options[:protocol] || 'http'
       @region =   options[:region] || 'us-east-1'
-      @aws_access_key_id =  options[:aws_access_key_id] || nil
-      @aws_secret_access_key = options[:aws_secret_access_key] || nil
-      @session_token = options[:session_token] || nil
-      @profile = options[:profile] || 'default'
-      @instance_cred_retries = options[:instance_profile_credentials_retries] || 0
-      @instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
+      @service_name = options[:service_name] || 'es'
+      aws_access_key_id =  options[:aws_access_key_id] || nil
+      aws_secret_access_key = options[:aws_secret_access_key] || nil
+      session_token = options[:session_token] || nil
+      profile = options[:profile] || 'default'
+      instance_cred_retries = options[:instance_profile_credentials_retries] || 0
+      instance_cred_timeout = options[:instance_profile_credentials_timeout] || 1
+
+      credential_config = CredentialConfig.new(aws_access_key_id, aws_secret_access_key, session_token, profile, instance_cred_retries, instance_cred_timeout, @region)
+      @credentials = Aws::CredentialProviderChain.new(credential_config).resolve
 
       if options[:proxy]
         options[:proxy] = manticore_proxy_hash(options[:proxy])
@@ -102,13 +110,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       key = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
                                                        :headers => params[:headers],:body => params[:body]})
 
-
-
-      credential_config = CredentialConfig.new(@aws_access_key_id, @aws_secret_access_key, @session_token, @profile, @instance_cred_retries, @instance_cred_timeout, @region)
-
-
-      credentials = Aws::CredentialProviderChain.new(credential_config).resolve
-      aws_signer = Aws::Signers::V4.new(credentials, 'es', @region )
+      aws_signer = Aws::Signers::V4.new(@credentials, @service_name, @region )
 
 
       signed_key =  aws_signer.sign(key)
@@ -126,7 +128,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # template installation. We might need a better story around this later
       # but for our current purposes this is correct
       if resp.code < 200 || resp.code > 299 && resp.code != 404
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body)
+        raise ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body, resp.body)
       end
 
       resp

data/lib/logstash/outputs/amazon_es/http_client/pool.rb

@@ -1,4 +1,8 @@
-module LogStash; module Outputs; class ElasticSearch; class HttpClient;
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+module LogStash; module Outputs; class AmazonElasticSearch; class HttpClient;
   class Pool
     class NoConnectionAvailableError < Error; end
     class BadResponseCodeError < Error
@@ -28,7 +32,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       end
     end
 
-    attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path
+    attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path, :skip_healthcheck
 
     ROOT_URI_PATH = '/'.freeze
 
@@ -40,6 +44,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       :resurrect_delay => 5,
       :sniffing => false,
       :sniffer_delay => 10,
+      :skip_healthcheck => false,
     }.freeze
 
     def initialize(logger, adapter, initial_urls=[], options={})
@@ -57,6 +62,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         @resurrect_delay = merged[:resurrect_delay]
         @sniffing = merged[:sniffing]
         @sniffer_delay = merged[:sniffer_delay]
+        @skip_healthcheck = merged[:skip_healthcheck]
       end
 
       # Used for all concurrent operations in this class
@@ -238,25 +244,30 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
 
     def healthcheck!
       # Try to keep locking granularity low such that we don't affect IO...
-      @state_mutex.synchronize { @url_info.select {|url,meta| meta[:state] != :alive } }.each do |url,meta|
+      @state_mutex.synchronize { @url_info.select { |url, meta| meta[:state] != :alive } }.each do |url, meta|
         begin
-          logger.info("Running health check to see if an Elasticsearch connection is working",
+          if skip_healthcheck == false
+            logger.info("Running health check to see if an Elasticsearch connection is working",
                         :healthcheck_url => url, :path => @healthcheck_path)
-          response = perform_request_to_url(url, :head, @healthcheck_path)
-          # If no exception was raised it must have succeeded!
-          logger.warn("Restored connection to ES instance", :url => url.sanitized.to_s)
-          # We reconnected to this node, check its ES version
-          es_version = get_es_version(url)
-          @state_mutex.synchronize do
-            meta[:version] = es_version
-            major = major_version(es_version)
-            if !@maximum_seen_major_version
-              @logger.info("ES Output version determined", :es_version => major)
-              set_new_major_version(major)
-            elsif major > @maximum_seen_major_version
-              @logger.warn("Detected a node with a higher major version than previously observed. This could be the result of an amazon_es cluster upgrade.", :previous_major => @maximum_seen_major_version, :new_major => major, :node_url => url)
-              set_new_major_version(major)
+            response = perform_request_to_url(url, :head, @healthcheck_path)
+            # If no exception was raised it must have succeeded!
+            logger.warn("Restored connection to ES instance", :url => url.sanitized.to_s)
+            # We reconnected to this node, check its ES version
+            es_version = get_es_version(url)
+            @state_mutex.synchronize do
+              meta[:version] = es_version
+              major = major_version(es_version)
+              if !@maximum_seen_major_version
+                @logger.info("ES Output version determined", :es_version => major)
+                set_new_major_version(major)
+              elsif major > @maximum_seen_major_version
+                @logger.warn("Detected a node with a higher major version than previously observed. This could be the result of an amazon_es cluster upgrade.", :previous_major => @maximum_seen_major_version, :new_major => major, :node_url => url)
+                set_new_major_version(major)
+              end
+              meta[:state] = :alive
             end
+          elsif skip_healthcheck == true
+            set_new_major_version(7)
             meta[:state] = :alive
           end
         rescue HostUnreachableError, BadResponseCodeError => e

data/lib/logstash/outputs/amazon_es/http_client.rb

@@ -1,3 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/outputs/amazon_es"
 require "cabin"
 require "base64"
@@ -7,24 +11,11 @@ require 'cgi'
 require 'zlib'
 require 'stringio'
 
-module LogStash; module Outputs; class ElasticSearch;
-  # This is a constant instead of a config option because
-  # there really isn't a good reason to configure it.
-  #
-  # The criteria used are:
-  # 1. We need a number that's less than 100MiB because ES
-  #    won't accept bulks larger than that.
-  # 2. It must be large enough to amortize the connection constant
-  #    across multiple requests.
-  # 3. It must be small enough that even if multiple threads hit this size
-  #    we won't use a lot of heap.
-  #
-  # We wound up agreeing that a number greater than 10 MiB and less than 100MiB
-  # made sense. We picked one on the lowish side to not use too much heap.
-  TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
-
+module LogStash; module Outputs; class AmazonElasticSearch;
   class HttpClient
-    attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
+    attr_reader :client, :options, :logger, :pool, :action_count, :recv_count, :max_bulk_bytes
+    
+  
     # This is here in case we use DEFAULT_OPTIONS in the future
     # DEFAULT_OPTIONS = {
     #   :setting => value
@@ -38,7 +29,7 @@ module LogStash; module Outputs; class ElasticSearch;
     # * `:user` - String. The user to use for authentication.
     # * `:password` - String. The password to use for authentication.
     # * `:timeout` - Float. A duration value, in seconds, after which a socket
-    #    operation or request will be aborted if not yet successfull
+    #    operation or request will be aborted if not yet successful
     # * `:client_settings` - a hash; see below for keys.
     #
     # The `client_settings` key is a has that can contain other settings:
@@ -54,6 +45,7 @@ module LogStash; module Outputs; class ElasticSearch;
       @metric = options[:metric]
       @bulk_request_metrics = @metric.namespace(:bulk_requests)
       @bulk_response_metrics = @bulk_request_metrics.namespace(:responses)
+      @max_bulk_bytes = options[:max_bulk_bytes] 
 
       # Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
       # @options = DEFAULT_OPTIONS.merge(options)
@@ -121,7 +113,7 @@ module LogStash; module Outputs; class ElasticSearch;
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-        if (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
+        if (body_stream.size + as_json.bytesize) > @max_bulk_bytes
           bulk_responses << bulk_send(body_stream) unless body_stream.size == 0
         end
         stream_writer.write(as_json)
@@ -152,7 +144,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       if response.code != 200
         url = ::LogStash::Util::SafeURI.new(response.final_url)
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+        raise ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
           response.code, url, body_stream.to_s, response.body
         )
       end
@@ -284,6 +276,8 @@ module LogStash; module Outputs; class ElasticSearch;
 
       adapter_options[:region] = options[:region]
 
+      adapter_options[:service_name] = options[:service_name]
+
       adapter_options[:port] = options[:port]
 
       adapter_options[:protocol] = options[:protocol]
@@ -292,7 +286,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
       adapter_options[:aws_secret_access_key] = options[:aws_secret_access_key]
 
-      adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
+      adapter_class = ::LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter
       adapter = adapter_class.new(@logger, adapter_options)
     end
     
@@ -306,11 +300,12 @@ module LogStash; module Outputs; class ElasticSearch;
         :healthcheck_path => options[:healthcheck_path],
         :resurrect_delay => options[:resurrect_delay],
         :url_normalizer => self.method(:host_to_url),
-        :metric => options[:metric]
+        :metric => options[:metric],
+        :skip_healthcheck => options[:skip_healthcheck],
       }
       pool_options[:scheme] = self.scheme if self.scheme
 
-      pool_class = ::LogStash::Outputs::ElasticSearch::HttpClient::Pool
+      pool_class = ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool
       full_urls = @options[:hosts].map {|h| host_to_url(h) }
       pool = pool_class.new(@logger, adapter, full_urls, pool_options)
       pool.start

data/lib/logstash/outputs/amazon_es/http_client_builder.rb

@@ -1,6 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require 'cgi'
 
-module LogStash; module Outputs; class ElasticSearch;
+module LogStash; module Outputs; class AmazonElasticSearch;
   module HttpClientBuilder
     def self.build(logger, hosts, params)
       client_settings = {
@@ -100,12 +104,16 @@ module LogStash; module Outputs; class ElasticSearch;
                                               :protocol => params["protocol"],
                                               :port => params["port"],
                                               :region => params["region"],
+                                              :service_name => params["service_name"],
                                               :aws_access_key_id => params["aws_access_key_id"],
-                                              :aws_secret_access_key => params["aws_secret_access_key"]))
+                                              :aws_secret_access_key => params["aws_secret_access_key"],
+                                              :max_bulk_bytes => params["max_bulk_bytes"],
+                                              :skip_healthcheck => params["skip_healthcheck"])
+                                              )
     end
 
     def self.create_http_client(options)
-      LogStash::Outputs::ElasticSearch::HttpClient.new(options)
+      LogStash::Outputs::AmazonElasticSearch::HttpClient.new(options)
     end
 
     def self.setup_ssl(logger, params)

data/lib/logstash/outputs/amazon_es/template_manager.rb

@@ -1,4 +1,8 @@
-module LogStash; module Outputs; class ElasticSearch
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+module LogStash; module Outputs; class AmazonElasticSearch
   class TemplateManager
     # To be mixed into the amazon_es plugin base
     def self.install_template(plugin)

data/lib/logstash/outputs/amazon_es.rb

@@ -1,3 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 # encoding: utf-8
 require "logstash/namespace"
 require "logstash/environment"
@@ -83,7 +87,7 @@ require "forwardable"
 # For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression` 
 # setting in their Logstash config file.
 #
-class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
+class LogStash::Outputs::AmazonElasticSearch < LogStash::Outputs::Base
   declare_threadsafe!
 
   require "logstash/outputs/amazon_es/http_client"
@@ -92,10 +96,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/amazon_es/common"
 
   # Protocol agnostic (i.e. non-http, non-java specific) configs go here
-  include(LogStash::Outputs::ElasticSearch::CommonConfigs)
+  include(LogStash::Outputs::AmazonElasticSearch::CommonConfigs)
 
   # Protocol agnostic methods
-  include(LogStash::Outputs::ElasticSearch::Common)
+  include(LogStash::Outputs::AmazonElasticSearch::Common)
 
   config_name "amazon_es"
 
@@ -121,11 +125,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # You can set the remote port as part of the host, or explicitly here as well
   config :port, :validate => :number, :default => 443
 
-  # Sets the protocol thats used to connect to elastisearch
+  # Sets the protocol thats used to connect to elasticsearch
   config :protocol, :validate => :string, :default => "https"
 
   #Signing specific details
   config :region, :validate => :string, :default => "us-east-1"
+
+  #Service name, default is `es`
+  config :service_name, :validate => :string, :default => "es"
   # Credential resolution logic works as follows:
   #
   # - User passed aws_access_key_id and aws_secret_access_key in aes configuration
@@ -248,9 +255,18 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Custom Headers to send on each request to amazon_es nodes
   config :custom_headers, :validate => :hash, :default => {}
 
+  #Max bulk size in bytes
+  config :max_bulk_bytes, :validate => :number, :default => 20 * 1024 * 1024
+
+  #Option for user to skip Healthcheck API for a host when set to True
+  config :skip_healthcheck, :validate => :boolean, :default => false
+
+  #Allow user to skip installing template when set to True
+  config :skip_template_installation, :validate => :boolean, :default => false
+
   def build_client
     params["metric"] = metric
-    @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
+    @client ||= ::LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
   end
 
   def close

data/logstash-output-amazon_es.gemspec

@@ -1,6 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-amazon_es'
-  s.version         = '6.4.1'
+  s.version         = '7.1.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Logstash Output to Amazon Elasticsearch Service"
   s.description     = "Output events to Amazon Elasticsearch Service with V4 signing"
@@ -26,9 +30,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'aws-sdk', '>= 2.3.22', '~> 2'
 
-  s.add_development_dependency 'logstash-codec-plain', '~> 0'
-  s.add_development_dependency 'logstash-devutils', '~> 0'
+  s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'logstash-devutils', "~> 1.3", ">= 1.3.1"
   s.add_development_dependency 'flores', '~> 0'
-  # Still used in some specs, we should remove this ASAP
-  s.add_development_dependency 'elasticsearch', '~> 0'
 end

data/spec/es_spec_helper.rb

@@ -1,6 +1,9 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/devutils/rspec/spec_helper"
 require 'manticore'
-require 'elasticsearch'
 
 # by default exclude secure_integration tests unless requested
 # normal integration specs are already excluded by devutils' spec helper
@@ -13,10 +16,6 @@ module ESHelper
     "127.0.0.1"
   end
 
-  def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
-  end
-
   def self.es_version
     RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
   end

data/spec/unit/http_client_builder_spec.rb

@@ -1,11 +1,15 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/amazon_es"
 require "logstash/outputs/amazon_es/http_client"
 require "logstash/outputs/amazon_es/http_client_builder"
 
-describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
+describe LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder do
   describe "auth setup with url encodable passwords" do
-    let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
+    let(:klass) { LogStash::Outputs::AmazonElasticSearch::HttpClientBuilder }
     let(:user) { "foo@bar"}
     let(:password) {"baz@blah" }
     let(:password_secured) do

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -1,7 +1,11 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/amazon_es/http_client"
 
-describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
+describe LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter do
   let(:logger) { Cabin::Channel.get }
   let(:options) { {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA',
                    :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'} }
@@ -34,12 +38,12 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
       uri_with_path = uri.clone
       uri_with_path.path = "/"
 
-      expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
+      expect(::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
         with(resp.code, uri_with_path, nil, resp.body).and_call_original
 
       expect do
         subject.perform_request(uri, :get, "/")
-      end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
+      end.to raise_error(::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError)
     end
   end
 

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -1,10 +1,14 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/amazon_es/http_client"
 require "json"
 
-describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
+describe LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool do
   let(:logger) { Cabin::Channel.get }
-  let(:adapter) { LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter.new(logger,
+  let(:adapter) { LogStash::Outputs::AmazonElasticSearch::HttpClient::ManticoreAdapter.new(logger,
                                                                                      {:aws_access_key_id => 'AAAAAAAAAAAAAAAAAAAA',
                                                                                               :aws_secret_access_key => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'}) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://localhost:9200")] }

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -1,8 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/amazon_es/http_client"
 require "java"
 
-describe LogStash::Outputs::ElasticSearch::HttpClient do
+describe LogStash::Outputs::AmazonElasticSearch::HttpClient do
   let(:ssl) { nil }
   let(:base_options) do
     opts = {
@@ -12,7 +16,8 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       :protocol => "http",
       :port => 9200,
       :aws_access_key_id => "AAAAAAAAAAAAAAAAAAAA",
-      :aws_secret_access_key => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+      :aws_secret_access_key => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+      :max_bulk_bytes => 20 * 1024 * 1024
     }
 
     if !ssl.nil? # Shortcut to set this
@@ -29,6 +34,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     let(:ipv6_hostname) { "[::1]" }
     let(:ipv4_hostname) { "127.0.0.1" }
     let(:port) { 9200 }
+    let(:max_bulk_bytes) { 20 * 1024 * 1024 }
     let(:protocol) {"http"}
     let(:aws_access_key_id) {"AAAAAAAAAAAAAAAAAAAA"}
     let(:aws_secret_access_key) {"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
@@ -163,7 +169,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     ]}
 
     context "if a message is over TARGET_BULK_BYTES" do
-      let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+      let(:target_bulk_bytes) { 20 * 1024 * 1024 }
       let(:message) { "a" * (target_bulk_bytes + 1) }
 
       it "should be handled properly" do
@@ -189,7 +195,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       end
 
       context "if one exceeds TARGET_BULK_BYTES" do
-        let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+        let(:target_bulk_bytes) { 20 * 1024 * 1024 }
         let(:message1) { "a" * (target_bulk_bytes + 1) }
         it "executes two bulk_send operations" do
           allow(subject).to receive(:join_bulk_responses)
@@ -201,7 +207,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
   end
 
   describe "sniffing" do
-    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
+    let(:client) { LogStash::Outputs::AmazonElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
 
     context "with sniffing enabled" do
       let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -1,9 +1,13 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/amazon_es/http_client"
 require "java"
 require "json"
 
-describe LogStash::Outputs::ElasticSearch::TemplateManager do
+describe LogStash::Outputs::AmazonElasticSearch::TemplateManager do
 
   describe ".default_template_path" do
     context "amazon_es 1.x" do

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -1,8 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require_relative "../../../spec/es_spec_helper"
 require "flores/random"
 require "logstash/outputs/amazon_es"
 
-describe LogStash::Outputs::ElasticSearch do
+describe LogStash::Outputs::AmazonElasticSearch do
   subject { described_class.new(options) }
   let(:options) { { "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
                     "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
@@ -222,7 +226,7 @@ describe LogStash::Outputs::ElasticSearch do
     context "429 errors" do
       let(:event) { ::LogStash::Event.new("foo" => "bar") }
       let(:error) do
-        ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+        ::LogStash::Outputs::AmazonElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
           429, double("url").as_null_object, double("request body"), double("response body")
         )
       end

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -1,3 +1,7 @@
+# SPDX-License-Identifier: Apache-2.0
+# See NOTICE for attribution details.
+# Modifications Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
 require "logstash/outputs/amazon_es"
 require_relative "../../../spec/es_spec_helper"
 
@@ -14,7 +18,7 @@ describe "whitelisting error types in expected behavior" do
                     "aws_access_key_id" => "AAAAAAAAAAAAAAAAAAAA",
                     "aws_secret_access_key" => "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"} }
 
-  subject { LogStash::Outputs::ElasticSearch.new(settings) }
+  subject { LogStash::Outputs::AmazonElasticSearch.new(settings) }
 
   before :each do
     allow(subject.logger).to receive(:warn)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-amazon_es
 version: !ruby/object:Gem::Version
-  version: 6.4.1
+  version: 7.1.0
 platform: java
 authors:
 - Amazon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-10 00:00:00.000000000 Z
+date: 2022-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -107,7 +107,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
@@ -115,7 +115,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -123,7 +123,10 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
   name: logstash-devutils
   prerelease: false
   type: :development
@@ -131,28 +134,17 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: flores
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
+        version: '1.3'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
-  name: elasticsearch
+  name: flores
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement