logstash-output-opensearch 1.1.0-java → 1.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de9f143d61a40ce521ac3500f2af934aa8a8d0da3cea50ac74b48f410ec54107
-  data.tar.gz: 46195137ee6c66e9fcc2b9a24f1ff474bf1444c4e5507560373dcf695014be64
+  metadata.gz: 2d7793b7f8cb45bd6d1d6a0893357497b1080fa46733e7806bd7bd6d0ec63ef3
+  data.tar.gz: d577bb202ad1e25d6ef34358dfad0d26cb22a0a579af4ba45343702c90768450
 SHA512:
-  metadata.gz: e96a1a3e78e07b6b74134483eb5ad6353c356a1f322a8b82222489241ecf3bcd59550885759217231973ac450fc3b284e38719c0e23bf80cfb54b8ee2dafdfad
-  data.tar.gz: 1ed119eefce3e7666f58e3cc57850c3809ec9dbbc7d6603f67808bd8d07a598c498580f5195c2aa361b4ee09cd6d49fb05170aa98d3faaf249a9c633876193e0
+  metadata.gz: a14b81ed9f97bd3c518227b5fc2e53cfbabab713dedd174aa128ac93674bb00e467e41eff9d11c60da3edd1e563f0da7e31fcdfc0d05f97f591adb93524fb82e
+  data.tar.gz: fb586673825bcc572f1e6b47fefa43540d6515e47542e2415838b56e7a9091586b16ebc07a2fb000955aab70c0459b7496564b88483064807c199711be42f5bf

data/CONTRIBUTING.md

@@ -6,6 +6,7 @@
     - [Documentation Changes](#documentation-changes)
     - [Contributing Code](#contributing-code)
 - [Developer Certificate of Origin](#developer-certificate-of-origin)
+- [License Headers](#license-headers)
 - [Review Process](#review-process)
 
 ## Contributing to logstash-output-opensearch
@@ -84,6 +85,31 @@ Signed-off-by: Jane Smith <jane.smith@email.com>
 ```
 You may type this line on your own when writing your commit messages.  However, if your user.name and user.email are set in your git configs, you can use `-s` or `– – signoff` to add the `Signed-off-by` line to the end of the commit message.
 
+## License Headers
+
+New files in your code contributions should contain the following license header. If you are modifying existing files with license headers, or including new files that already have license headers, do not remove or modify them without guidance.
+
+### Java
+
+```
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+```
+
+### Python
+```
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+```
+
+### Shell
+```
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+```
+
 ## Review Process
 
 We deeply appreciate everyone who takes the time to make a contribution.  We will review all contributions as quickly as possible.  As a reminder, [opening an issue](https://github.com/opensearch-project/logstash-output-opensearch/issues/new/choose) discussing your change before you make it is the best way to smooth the PR process.  This will prevent a rejection because someone else is already working on the problem, or because the solution is incompatible with the architectural direction.

data/README.md

@@ -15,7 +15,8 @@
 ## Project Resources
 
 * [Project Website](https://opensearch.org/)
-* [Documentation](https://opensearch.org/)
+* [Documentation](https://opensearch.org/docs/clients/logstash/index/)
+* [Developer Guide](DEVELOPER_GUIDE.md)
 * Need help? Try [Forums](https://discuss.opendistrocommunity.dev/)
 * [Project Principles](https://opensearch.org/#principles)
 * [Contributing to OpenSearch](CONTRIBUTING.md)

data/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb

@@ -7,12 +7,31 @@
 #  Modifications Copyright OpenSearch Contributors. See
 #  GitHub history for details.
 
-require 'manticore'
+require 'aws-sdk-core'
 require 'cgi'
+require 'manticore'
+require 'uri'
 
 module LogStash; module Outputs; class OpenSearch; class HttpClient;
+  AWS_DEFAULT_PORT = 443
+  AWS_DEFAULT_PROFILE = 'default'
+  AWS_DEFAULT_PROFILE_CREDENTIAL_RETRY = 0
+  AWS_DEFAULT_PROFILE_CREDENTIAL_TIMEOUT = 1
+  AWS_DEFAULT_REGION = 'us-east-1'
+  AWS_IAM_AUTH_TYPE = "aws_iam"
+  AWS_SERVICE = 'es'
+  BASIC_AUTH_TYPE = 'basic'
   DEFAULT_HEADERS = { "content-type" => "application/json" }
-  
+
+  AWSIAMCredential = Struct.new(
+    :access_key_id,
+    :secret_access_key,
+    :session_token,
+    :profile,
+    :instance_profile_credentials_retries,
+    :instance_profile_credentials_timeout,
+    :region)
+
   class ManticoreAdapter
     attr_reader :manticore, :logger
 
@@ -27,13 +46,65 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       options[:cookies] = false
 
       @client_params = {:headers => DEFAULT_HEADERS.merge(options[:headers] || {})}
-      
+      @type = get_auth_type(options) || nil
+
+      if @type == AWS_IAM_AUTH_TYPE
+        aws_iam_auth_initialization(options)
+      elsif @type == BASIC_AUTH_TYPE
+        basic_auth_initialization(options)
+      end
+
       if options[:proxy]
         options[:proxy] = manticore_proxy_hash(options[:proxy])
       end
       
       @manticore = ::Manticore::Client.new(options)
     end
+
+    def get_auth_type(options)
+      if options[:auth_type] != nil
+        options[:auth_type]["type"]
+      end
+    end
+
+    def aws_iam_auth_initialization(options)
+      aws_access_key_id =  options[:auth_type]["aws_access_key_id"] || nil
+      aws_secret_access_key = options[:auth_type]["aws_secret_access_key"] || nil
+      session_token = options[:auth_type]["session_token"] || nil
+      profile = options[:auth_type]["profile"] || AWS_DEFAULT_PROFILE
+      instance_cred_retries = options[:auth_type]["instance_profile_credentials_retries"] || AWS_DEFAULT_PROFILE_CREDENTIAL_RETRY
+      instance_cred_timeout = options[:auth_type]["instance_profile_credentials_timeout"] || AWS_DEFAULT_PROFILE_CREDENTIAL_TIMEOUT
+      region = options[:auth_type]["region"] || AWS_DEFAULT_REGION
+      set_aws_region(region)
+
+      credential_config = AWSIAMCredential.new(aws_access_key_id, aws_secret_access_key, session_token, profile, instance_cred_retries, instance_cred_timeout, region)
+      @credentials = Aws::CredentialProviderChain.new(credential_config).resolve
+    end
+
+    def basic_auth_initialization(options)
+      set_user_password(options)
+    end
+
+    def set_aws_region(region)
+      @region = region
+    end
+
+    def get_aws_region()
+      @region
+    end
+
+    def set_user_password(options)
+      @user = options[:auth_type]["user"]
+      @password = options[:auth_type]["password"]
+    end
+
+    def get_user()
+      @user
+    end
+
+    def get_password()
+      @password
+    end
     
     # Transform the proxy option to a hash. Manticore's support for non-hash
     # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
@@ -61,6 +132,8 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       params = (params || {}).merge(@client_params) { |key, oldval, newval|
         (oldval.is_a?(Hash) && newval.is_a?(Hash)) ? oldval.merge(newval) : newval
       }
+
+      params[:headers] = params[:headers].clone
       params[:body] = body if body
 
       if url.user
@@ -71,9 +144,16 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
           :password => CGI.unescape(url.password), 
           :eager => true 
         }
+      elsif @type == BASIC_AUTH_TYPE
+        add_basic_auth_to_params(params)
       end
 
       request_uri = format_url(url, path)
+
+      if @type == AWS_IAM_AUTH_TYPE
+        sign_aws_request(request_uri, path, method, params)
+      end
+
       request_uri_as_string = remove_double_escaping(request_uri.to_s)
       resp = @manticore.send(method.downcase, request_uri_as_string, params)
 
@@ -92,6 +172,23 @@ module LogStash; module Outputs; class OpenSearch; class HttpClient;
       resp
     end
 
+    def sign_aws_request(request_uri, path, method, params)
+      url = URI::HTTPS.build({:host=>URI(request_uri.to_s).host, :port=>AWS_DEFAULT_PORT.to_s, :path=>path})
+      key = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
+                                                         :headers => params[:headers],:body => params[:body]})
+      aws_signer = Aws::Signers::V4.new(@credentials,  AWS_SERVICE, get_aws_region )
+      signed_key =  aws_signer.sign(key)
+      params[:headers] =  params[:headers].merge(signed_key.headers)
+    end
+
+    def add_basic_auth_to_params(params)
+      params[:auth] = {
+        :user => get_user(),
+        :password => get_password(),
+        :eager => true
+      }
+    end
+
     # Returned urls from this method should be checked for double escaping.
     def format_url(url, path_and_query=nil)
       request_uri = url.clone

data/lib/logstash/outputs/opensearch/http_client.rb

@@ -15,23 +15,9 @@ require 'zlib'
 require 'stringio'
 
 module LogStash; module Outputs; class OpenSearch;
-  # This is a constant instead of a config option because
-  # there really isn't a good reason to configure it.
-  #
-  # The criteria used are:
-  # 1. We need a number that's less than 100MiB because OpenSearch
-  #    won't accept bulks larger than that.
-  # 2. It must be large enough to amortize the connection constant
-  #    across multiple requests.
-  # 3. It must be small enough that even if multiple threads hit this size
-  #    we won't use a lot of heap.
-  #
-  # We wound up agreeing that a number greater than 10 MiB and less than 100MiB
-  # made sense. We picked one on the lowish side to not use too much heap.
-  TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
-
   class HttpClient
-    attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
+    attr_reader :client, :options, :logger, :pool, :action_count, :recv_count, :target_bulk_bytes
+
     # This is here in case we use DEFAULT_OPTIONS in the future
     # DEFAULT_OPTIONS = {
     #   :setting => value
@@ -41,13 +27,14 @@ module LogStash; module Outputs; class OpenSearch;
     # The `options` is a hash where the following symbol keys have meaning:
     #
     # * `:hosts` - array of String. Set a list of hosts to use for communication.
-    # * `:port` - number. set the port to use to communicate with OpenSearch
     # * `:user` - String. The user to use for authentication.
     # * `:password` - String. The password to use for authentication.
     # * `:timeout` - Float. A duration value, in seconds, after which a socket
     #    operation or request will be aborted if not yet successfull
+    # * `:auth_type` - hash of String. It contains the type of authentication
+    #     and it's respective credentials
     # * `:client_settings` - a hash; see below for keys.
-    #
+
     # The `client_settings` key is a has that can contain other settings:
     #
     # * `:ssl` - Boolean. Enable or disable SSL/TLS.
@@ -72,6 +59,8 @@ module LogStash; module Outputs; class OpenSearch;
       # mutex to prevent requests and sniffing to access the
       # connection pool at the same time
       @bulk_path = @options[:bulk_path]
+
+      @target_bulk_bytes = @options[:target_bulk_bytes]
     end
 
     def build_url_template
@@ -104,7 +93,6 @@ module LogStash; module Outputs; class OpenSearch;
     def bulk(actions)
       @action_count ||= 0
       @action_count += actions.size
-
       return if actions.empty?
 
       bulk_actions = actions.collect do |action, args, source|
@@ -131,7 +119,7 @@ module LogStash; module Outputs; class OpenSearch;
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-        if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
+        if (stream_writer.pos + as_json.bytesize) > @target_bulk_bytes && stream_writer.pos > 0
           stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
           logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
                        :action_count => batch_actions.size,
@@ -324,6 +312,8 @@ module LogStash; module Outputs; class OpenSearch;
 
       adapter_options[:headers] = client_settings[:headers] if client_settings[:headers]
 
+      adapter_options[:auth_type] = options[:auth_type]
+
       adapter_class = ::LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter
       adapter = adapter_class.new(@logger, adapter_options)
     end
@@ -400,7 +390,7 @@ module LogStash; module Outputs; class OpenSearch;
 
     def template_endpoint
       # TODO: Check Version < 7.8 and use index template for >= 7.8 & OpenSearch
-      # https://docs-beta.opensearch.org/opensearch/index-templates/
+      # https://opensearch.org/docs/opensearch/index-templates/
       '_template'
     end
 

data/lib/logstash/outputs/opensearch/http_client_builder.rb

@@ -35,6 +35,7 @@ module LogStash; module Outputs; class OpenSearch;
       end
 
       common_options[:timeout] = params["timeout"] if params["timeout"]
+      common_options[:target_bulk_bytes] = params["target_bulk_bytes"]
 
       if params["path"]
         client_settings[:path] = dedup_slashes("/#{params["path"]}/")
@@ -106,7 +107,10 @@ module LogStash; module Outputs; class OpenSearch;
       }
       common_options.merge! update_options if params["action"] == 'update'
 
-      create_http_client(common_options.merge(:hosts => hosts, :logger => logger))
+      create_http_client(common_options.merge(:hosts => hosts,
+                                              :logger => logger,
+                                              :auth_type => params["auth_type"]
+                                              ))
     end
 
     def self.create_http_client(options)

data/lib/logstash/outputs/opensearch.rb

@@ -95,7 +95,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
   #   would use the foo field for the action
   #
-  # For more details on actions, check out the https://docs-beta.opensearch.org/opensearch/rest-api/bulk/[OpenSearch bulk API documentation]
+  # For more details on actions, check out the https://opensearch.org/docs/opensearch/rest-api/document-apis/bulk/[OpenSearch bulk API documentation]
   config :action, :validate => :string, :default => "index"
 
   # The index to write events to. This can be dynamic using the `%{foo}` syntax.

data/lib/logstash/plugin_mixins/opensearch/api_configs.rb

@@ -20,6 +20,18 @@ module LogStash; module PluginMixins; module OpenSearch
         # Password to authenticate to a secure OpenSearch cluster
         :password => { :validate => :password },
 
+        # if auth_type is "aws_iam" then
+        #   Credential resolution logic works as follows :
+        #
+        # - User passed aws_access_key_id and aws_secret_access_key in opensearch configuration
+        # - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+        #   (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET),
+        #   or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK)
+        # - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
+        # - Instance profile credentials delivered through the Amazon EC2 metadata service
+        # - type in auth_type specifies the type of authentication
+        :auth_type => { },
+
         # The document ID for the index. Useful for overwriting existing entries in
         # OpenSearch with the same ID.
         :document_id => { :validate => :string },
@@ -34,6 +46,20 @@ module LogStash; module PluginMixins; module OpenSearch
         # this defaults to a concatenation of the path parameter and "_bulk"
         :bulk_path => { :validate => :string },
 
+        # Maximum number of bytes in bulk requests
+        # The criteria for deciding the default value of target_bulk_bytes is:
+        # 1. We need a number that's less than 10MiB because OpenSearch is commonly
+        #    configured (particular in AWS Opensearch Service) to not accept
+        #    bulks larger than that.
+        # 2. It must be large enough to amortize the connection constant
+        #    across multiple requests.
+        # 3. It must be small enough that even if multiple threads hit this size
+        #    we won't use a lot of heap.
+        :target_bulk_bytes => {
+            :validate => :number,
+            :default => 9 * 1024 * 1024 # 9MiB
+        },
+
         # Pass a set of key value pairs as the URL query string. This query string is added
         # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
         # urls that already have query strings, the one specified here will be appended.

data/logstash-output-opensearch.gemspec

@@ -1,6 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+signing_key_path = "gem-private_key.pem"
+
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.1.0'
+  s.version         = '1.2.0'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -18,8 +29,10 @@ Gem::Specification.new do |s|
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
-  s.cert_chain  = ['certs/opensearch-rubygems.pem']
-  s.signing_key = File.expand_path("private_key.pem") if $0 =~ /gem\z/
+  if $PROGRAM_NAME.end_with?("gem") && ARGV == ["build", __FILE__] && File.exist?(signing_key_path)
+    s.signing_key = signing_key_path
+    s.cert_chain  = ['certs/opensearch-rubygems.pem']
+  end
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = {
@@ -32,9 +45,11 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
+  s.add_runtime_dependency 'aws-sdk',  '>= 2.11.632', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
   s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'opensearch-ruby'
 end

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -33,8 +33,8 @@ describe "indexing with http_compression turned on", :integration => true do
   }
   subject { LogStash::Outputs::OpenSearch.new(config) }
 
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
+  let(:opensearch_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{opensearch_url}/#{index}"}
   let(:http_client_options) { {} }
   let(:http_client) do
     Manticore::Client.new(http_client_options)
@@ -49,7 +49,7 @@ describe "indexing with http_compression turned on", :integration => true do
     it "ships events" do
       subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{opensearch_url}/_refresh").call
 
       response = http_client.get("#{index_url}/_count?q=*")
       result = LogStash::Json.load(response.body)

data/spec/integration/outputs/create_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "client create actions", :integration => true do
   require "logstash/outputs/opensearch"
 
-  def get_es_output(action, id, version=nil, version_type=nil)
+  def get_output(action, id, version=nil, version_type=nil)
     settings = {
       "manage_template" => true,
       "index" => "logstash-create",
@@ -37,7 +37,7 @@ describe "client create actions", :integration => true do
 
   context "when action => create" do
     it "should create new documents with or without id" do
-      subject = get_es_output("create", "id123")
+      subject = get_output("create", "id123")
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       @client.indices.refresh
@@ -49,27 +49,27 @@ describe "client create actions", :integration => true do
     end
 
     it "should allow default (internal) version" do
-      subject = get_es_output("create", "id123", 43)
+      subject = get_output("create", "id123", 43)
       subject.register
     end
 
     it "should allow internal version" do
-      subject = get_es_output("create", "id123", 43, "internal")
+      subject = get_output("create", "id123", 43, "internal")
       subject.register
     end
 
     it "should not allow external version" do
-      subject = get_es_output("create", "id123", 43, "external")
+      subject = get_output("create", "id123", 43, "external")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gt version" do
-      subject = get_es_output("create", "id123", 43, "external_gt")
+      subject = get_output("create", "id123", 43, "external_gt")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gte version" do
-      subject = get_es_output("create", "id123", 43, "external_gte")
+      subject = get_output("create", "id123", 43, "external_gte")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end

data/spec/integration/outputs/delete_spec.rb

@@ -14,15 +14,15 @@ require "logstash/outputs/opensearch"
 describe "Versioned delete", :integration => true do
   require "logstash/outputs/opensearch"
 
-  let(:es) { get_client }
+  let(:client) { get_client }
 
   before :each do
     # Delete all templates first.
     # Clean ES of data before we start.
-    es.indices.delete_template(:name => "*")
+    client.indices.delete_template(:name => "*")
     # This can fail if there are no indexes, ignore failure.
-    es.indices.delete(:index => "*") rescue nil
-    es.indices.refresh
+    client.indices.delete(:index => "*") rescue nil
+    client.indices.refresh
   end
 
   context "when delete only" do
@@ -48,12 +48,12 @@ describe "Versioned delete", :integration => true do
     it "should ignore non-monotonic external version updates" do
       id = "ev2"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r2 = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r2['_version']).to eq(99)
       expect(r2['_source']['message']).to eq('foo')
     end
@@ -61,12 +61,12 @@ describe "Versioned delete", :integration => true do
     it "should commit monotonic external version updates" do
       id = "ev3"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      expect { client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(OpenSearch::Transport::Transport::Errors::NotFound)
     end
   end
 end

data/spec/integration/outputs/index_spec.rb

@@ -10,8 +10,7 @@
 require_relative "../../../spec/opensearch_spec_helper"
 require "logstash/outputs/opensearch"
 
-describe "TARGET_BULK_BYTES", :integration => true do
-  let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+describe "target_bulk_bytes", :integration => true do
   let(:event_count) { 1000 }
   let(:events) { event_count.times.map { event }.to_a }
   let(:config) {
@@ -32,11 +31,11 @@ describe "TARGET_BULK_BYTES", :integration => true do
   end
 
   describe "batches that are too large for one" do
-    let(:event) { LogStash::Event.new("message" => "a " * (((target_bulk_bytes/2) / event_count)+1)) }
+    let(:event) { LogStash::Event.new("message" => "a " * (((subject.client.target_bulk_bytes/2) / event_count)+1)) }
 
     it "should send in two batches" do
       expect(subject.client).to have_received(:bulk_send).twice do |payload|
-        expect(payload.size).to be <= target_bulk_bytes
+        expect(payload.size).to be <= subject.client.target_bulk_bytes
       end
     end
 
@@ -47,7 +46,7 @@ describe "TARGET_BULK_BYTES", :integration => true do
 
       it "should send in one batch" do
         expect(subject.client).to have_received(:bulk_send).once do |payload|
-          expect(payload.size).to be <= target_bulk_bytes
+          expect(payload.size).to be <= subject.client.target_bulk_bytes
         end
       end
     end
@@ -63,8 +62,8 @@ describe "indexing" do
   let(:events) { event_count.times.map { event }.to_a }
   subject { LogStash::Outputs::OpenSearch.new(config) }
   
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
+  let(:opensearch_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{opensearch_url}/#{index}"}
   let(:http_client_options) { {} }
   let(:http_client) do
     Manticore::Client.new(http_client_options)
@@ -79,7 +78,7 @@ describe "indexing" do
     it "ships events" do
       subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{opensearch_url}/_refresh").call
 
       response = http_client.get("#{index_url}/_count?q=*")
       result = LogStash::Json.load(response.body)
@@ -136,7 +135,7 @@ describe "indexing" do
   describe "a secured indexer", :secure_integration => true do
     let(:user) { "admin" }
     let(:password) { "admin" }
-    let(:es_url) {"https://integration:9200"}
+    let(:opensearch_url) {"https://integration:9200"}
     let(:config) do
       {
         "hosts" => ["integration:9200"],
@@ -161,4 +160,41 @@ describe "indexing" do
     end
     it_behaves_like("an indexer", true)
   end
+
+  describe "a basic auth secured indexer", :secure_integration => true do
+    let(:options) { {
+      :auth_type => {
+        "type"=>"basic",
+        "user" => "admin",
+        "password" => "admin"}
+    } }
+    let(:user) {options[:auth_type]["user"]}
+    let(:password) {options[:auth_type]["password"]}
+    let(:opensearch_url) {"https://integration:9200"}
+    let(:config) do
+      {
+        "hosts" => ["integration:9200"],
+        "auth_type" => {
+          "type"=>"basic",
+          "user" => user,
+          "password" => password},
+        "ssl" => true,
+        "ssl_certificate_verification" => false,
+        "index" => index
+      }
+    end
+    let(:http_client_options) do
+      {
+        :auth => {
+          :user => user,
+          :password => password
+        },
+        :ssl => {
+          :enabled => true,
+          :verify => false
+        }
+      }
+    end
+    it_behaves_like("an indexer", true)
+  end
 end

data/spec/integration/outputs/index_version_spec.rb

@@ -13,15 +13,15 @@ require "logstash/outputs/opensearch"
 describe "Versioned indexing", :integration => true do
   require "logstash/outputs/opensearch"
 
-  let(:es) { get_client }
+  let(:client) { get_client }
 
   before :each do
     # Delete all templates first.
     # Clean OpenSearch of data before we start.
-    es.indices.delete_template(:name => "*")
+    client.indices.delete_template(:name => "*")
     # This can fail if there are no indexes, ignore failure.
-    es.indices.delete(:index => "*") rescue nil
-    es.indices.refresh
+    client.indices.delete(:index => "*") rescue nil
+    client.indices.refresh
   end
 
   context "when index only" do
@@ -46,11 +46,11 @@ describe "Versioned indexing", :integration => true do
 
       it "should default to OpenSearch version" do
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r["_version"]).to eq(1)
         expect(r["_source"]["message"]).to eq('foo')
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r2["_version"]).to eq(2)
         expect(r2["_source"]["message"]).to eq('foobar')
       end
@@ -74,7 +74,7 @@ describe "Versioned indexing", :integration => true do
       it "should respect the external version" do
         id = "ev1"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
       end
@@ -82,12 +82,12 @@ describe "Versioned indexing", :integration => true do
       it "should ignore non-monotonic external version updates" do
         id = "ev2"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(99)
         expect(r2["_source"]["message"]).to eq('foo')
       end
@@ -95,12 +95,12 @@ describe "Versioned indexing", :integration => true do
       it "should commit monotonic external version updates" do
         id = "ev3"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(100)
         expect(r2["_source"]["message"]).to eq('foo')
       end

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -37,8 +37,6 @@ describe "Ingest pipeline execution behavior", :integration => true do
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
-
     # Clean OpenSearch of data before we start.
     @client = get_client
     @client.indices.delete_template(:name => "*")

data/spec/integration/outputs/metrics_spec.rb

@@ -24,8 +24,6 @@ describe "metrics", :integration => true do
   let(:document_level_metrics) { subject.instance_variable_get(:@document_level_metrics) }
 
   before :each do
-    require "elasticsearch"
-
     # Clean OpenSearch of data before we start.
     @client = get_client
     clean(@client)

data/spec/integration/outputs/no_opensearch_on_startup_spec.rb

@@ -27,7 +27,6 @@ describe "opensearch is down on startup", :integration => true do
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
     allow(Stud).to receive(:stoppable_sleep)
 
     # Clean OpenSearch of data before we start.

data/spec/integration/outputs/painless_update_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
   require "logstash/outputs/opensearch"
 
-  def get_es_output( options={} )
+  def get_output( options={} )
     settings = {
       "manage_template" => true,
       "index" => "logstash-update",
@@ -42,7 +42,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
   context "scripted updates" do
 
     it "should increment a counter with event/doc 'count' variable with inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "123",
         'script' => 'ctx._source.counter += params.event.counter',
         'script_type' => 'inline'
@@ -54,7 +54,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "123",
         'doc_as_upsert' => true,
         'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
@@ -67,7 +67,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "456",
         'doc_as_upsert' => true,
         'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
@@ -90,7 +90,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
         plugin_parameters.merge!('script_lang' => '')
 
-        subject = get_es_output(plugin_parameters)
+        subject = get_output(plugin_parameters)
         subject.register
         subject.multi_receive([LogStash::Event.new("count" => 4 )])
         r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -101,7 +101,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
   context "when update with upsert" do
     it "should create new documents with provided upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject = get_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -109,7 +109,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should create new documents with event/doc as upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -117,7 +117,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should fail on documents with event/doc as upsert at external version" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end
@@ -126,7 +126,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
     context 'with an inline script' do
       it "should create new documents with upsert content" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
+        subject = get_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
         subject.register
 
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
@@ -135,7 +135,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
       end
 
       it "should create new documents with event/doc as script params" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
+        subject = get_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 1)])
         @client.indices.refresh

data/spec/integration/outputs/retry_spec.rb

@@ -53,7 +53,6 @@ describe "failures in bulk class expected behavior", :integration => true do
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
     allow(Stud).to receive(:stoppable_sleep)
 
     # Clean OpenSearch of data before we start.

data/spec/integration/outputs/sniffer_spec.rb

@@ -15,9 +15,9 @@ require "socket"
 describe "pool sniffer", :integration => true do
   let(:logger) { Cabin::Channel.get }
   let(:adapter) { LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter.new(logger) }
-  let(:es_host) { get_host_port.split(":").first }
+  let(:opensearch_host) { get_host_port.split(":").first }
   let(:es_port) { get_host_port.split(":").last }
-  let(:es_ip) { IPSocket.getaddress(es_host) }
+  let(:opensearch_ip) { IPSocket.getaddress(opensearch_host) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
   let(:options) do
     {

data/spec/integration/outputs/templates_spec.rb

@@ -21,8 +21,6 @@ describe "index template expected behavior", :integration => true do
   end
 
   before :each do
-    # Delete all templates first.
-    require "elasticsearch"
 
     # Clean OpenSearch of data before we start.
     @client = get_client

data/spec/integration/outputs/update_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "Update actions without scripts", :integration => true do
   require "logstash/outputs/opensearch"
 
-  def get_es_output( options={} )
+  def get_output( options={} )
     settings = {
       "manage_template" => true,
       "index" => "logstash-update",
@@ -40,20 +40,20 @@ describe "Update actions without scripts", :integration => true do
   end
 
   it "should fail without a document_id" do
-    subject = get_es_output
+    subject = get_output
     expect { subject.register }.to raise_error(LogStash::ConfigurationError)
   end
 
   context "when update only" do
     it "should not create new document" do
-      subject = get_es_output({ 'document_id' => "456" } )
+      subject = get_output({ 'document_id' => "456" } )
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(OpenSearch::Transport::Transport::Errors::NotFound)
     end
 
     it "should update existing document" do
-      subject = get_es_output({ 'document_id' => "123" })
+      subject = get_output({ 'document_id' => "123" })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -63,7 +63,7 @@ describe "Update actions without scripts", :integration => true do
     # The es ruby client treats the data field differently. Make sure this doesn't
     # raise an exception
     it "should update an existing document that has a 'data' field" do
-      subject = get_es_output({ 'document_id' => "123" })
+      subject = get_output({ 'document_id' => "123" })
       subject.register
       subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -72,27 +72,27 @@ describe "Update actions without scripts", :integration => true do
     end
 
     it "should allow default (internal) version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+      subject = get_output({ 'document_id' => "123", "version" => "99" })
       subject.register
     end
 
     it "should allow internal version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
       subject.register
     end
 
     it "should not allow external version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gt version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gte version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
@@ -100,7 +100,7 @@ describe "Update actions without scripts", :integration => true do
 
   context "when update with upsert" do
     it "should create new documents with provided upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject = get_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -108,7 +108,7 @@ describe "Update actions without scripts", :integration => true do
     end
 
     it "should create new documents with event/doc as upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -116,7 +116,7 @@ describe "Update actions without scripts", :integration => true do
     end
 
     it "should fail on documents with event/doc as upsert at external version" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end

data/spec/opensearch_spec_helper.rb

@@ -9,7 +9,7 @@
 
 require_relative './spec_helper'
 
-require 'elasticsearch'
+require 'opensearch'
 
 require 'json'
 require 'cabin'
@@ -24,7 +24,7 @@ module OpenSearchHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+    OpenSearch::Client.new(:hosts => [get_host_port])
   end
 
   def doc_type

data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb

@@ -25,13 +25,13 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
   it "should implement host unreachable exceptions" do
     expect(subject.host_unreachable_exceptions).to be_a(Array)
   end
-  
+
   describe "auth" do
     let(:user) { "myuser" }
     let(:password) { "mypassword" }
     let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
     let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
-    
+
     it "should convert the auth to params" do
       resp = double("response")
       allow(resp).to receive(:call)
@@ -39,7 +39,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
       
       expected_uri = noauth_uri.clone
       expected_uri.path = "/"
-      
+
       expect(subject.manticore).to receive(:get).
         with(expected_uri.to_s, {
           :headers => {"content-type" => "application/json"},
@@ -54,6 +54,76 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
     end
   end
 
+  describe "aws_iam" do
+    let(:options) { {
+      :auth_type => {
+        "type"=>"aws_iam",
+        "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
+        "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
+    } }
+    subject { described_class.new(logger, options) }
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+    let(:sign_aws_request) {  }
+
+    it "should validate AWS IAM credentials initialization" do
+      expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
+    end
+
+    it "should validate signing aws request" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(200)
+      allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+
+      expected_uri = uri.clone
+      expected_uri.path = "/"
+
+      expect(subject.manticore).to receive(:get).
+        with(expected_uri.to_s, {
+          :headers => {"content-type"=> "application/json"}
+        }
+        ).and_return resp
+
+      expect(subject).to receive(:sign_aws_request)
+      subject.perform_request(uri, :get, "/")
+    end
+  end
+
+  describe "basic_auth" do
+    let(:options) { {
+      :auth_type => {
+        "type"=>"basic",
+        "user" => "myuser",
+        "password" => "mypassword"}
+    } }
+    subject { described_class.new(logger, options) }
+    let(:user) {options[:auth_type]["user"]}
+    let(:password) {options[:auth_type]["password"]}
+    let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+
+    it "should validate master credentials with type as 'basic_auth'" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(200)
+
+      expected_uri = noauth_uri.clone
+      expected_uri.path = "/"
+
+      expect(subject.manticore).to receive(:get).
+        with(expected_uri.to_s, {
+          :headers => {"content-type" => "application/json"},
+          :auth => {
+            :user => user,
+            :password => password,
+            :eager => true
+          }
+        }).and_return resp
+
+      subject.perform_request(uri, :get, "/")
+    end
+  end
+
   describe "bad response codes" do
     let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
 

data/spec/unit/outputs/opensearch/http_client_spec.rb

@@ -17,6 +17,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
     opts = {
       :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
       :logger => Cabin::Channel.get,
+      :target_bulk_bytes => 9_000_000,
       :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
     }
 
@@ -226,8 +227,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
           end
         end
 
-        context "if a message is over TARGET_BULK_BYTES" do
-          let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+        context "if a message is over target_bulk_bytes" do
+          let(:target_bulk_bytes) { subject.target_bulk_bytes }
           let(:message) { "a" * (target_bulk_bytes + 1) }
 
           it "should be handled properly" do
@@ -256,8 +257,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
             s = subject.send(:bulk, actions)
           end
 
-          context "if one exceeds TARGET_BULK_BYTES" do
-            let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+          context "if one exceeds target_bulk_bytes" do
+            let(:target_bulk_bytes) { subject.target_bulk_bytes }
             let(:message1) { "a" * (target_bulk_bytes + 1) }
             it "executes two bulk_send operations" do
               allow(subject).to receive(:join_bulk_responses)

data/spec/unit/outputs/opensearch_spec.rb

@@ -325,7 +325,7 @@ describe LogStash::Outputs::OpenSearch do
   end
 
   context '413 errors' do
-    let(:payload_size) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES + 1024 }
+    let(:payload_size) { subject.client.target_bulk_bytes + 1024 }
     let(:event) { ::LogStash::Event.new("message" => ("a" * payload_size ) ) }
 
     let(:logger_stub) { double("logger").as_null_object }
@@ -357,7 +357,7 @@ describe LogStash::Outputs::OpenSearch do
 
       expect(logger_stub).to have_received(:warn)
                                  .with(a_string_matching(/413 Payload Too Large/),
-                                       hash_including(:action_count => 1, :content_length => a_value > 20_000_000))
+                                       hash_including(:action_count => 1, :content_length => a_value > subject.client.target_bulk_bytes))
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: java
 authors:
 - Elastic
@@ -31,7 +31,7 @@ cert_chain:
   yXikuH6LEVykA8pgOcB9gKsB2/zMd2ZlSj2monM8Qw9EfB14ZSDTYS8VYuwWCeF0
   eFmXXk0ufQFKl1Yll7quHkmQ0PzKkvXTpONBT6qPkXE=
   -----END CERTIFICATE-----
-date: 2021-09-21 00:00:00.000000000 Z
+date: 2021-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +107,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.11.632
+  name: aws-sdk
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.11.632
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -163,6 +183,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: opensearch-ruby
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gem. This gem
   is not a stand-alone program