logstash-output-opensearch 1.0.0-java → 1.3.0-java

data/lib/logstash/outputs/opensearch.rb

@@ -68,7 +68,6 @@ require "forwardable"
 class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   declare_threadsafe!
 
-  require "logstash/outputs/opensearch/distribution_checker"
   require "logstash/outputs/opensearch/http_client"
   require "logstash/outputs/opensearch/http_client_builder"
   require "logstash/plugin_mixins/opensearch/api_configs"
@@ -79,7 +78,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   include(LogStash::PluginMixins::OpenSearch::Common)
 
   # ecs_compatibility option, provided by Logstash core or the support adapter.
-  include(LogStash::PluginMixins::ECSCompatibilitySupport)
+  include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
 
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::OpenSearch::APIConfigs)
@@ -96,7 +95,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
   #   would use the foo field for the action
   #
-  # For more details on actions, check out the https://docs-beta.opensearch.org/opensearch/rest-api/bulk/[OpenSearch bulk API documentation]
+  # For more details on actions, check out the https://opensearch.org/docs/opensearch/rest-api/document-apis/bulk/[OpenSearch bulk API documentation]
   config :action, :validate => :string, :default => "index"
 
   # The index to write events to. This can be dynamic using the `%{foo}` syntax.
@@ -218,7 +217,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
 
     @logger.info("New OpenSearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
 
-    @client = build_client(DistributionChecker.new(@logger))
+    @client = build_client
 
     @after_successful_connection_thread = after_successful_connection do
       begin
@@ -420,7 +419,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
     when :disabled
       @default_index = "logstash-%{+yyyy.MM.dd}"
       @default_template_name = 'logstash'
-    when :v1
+    when :v1, :v8
       @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
       @default_template_name = 'ecs-logstash'
     else

data/lib/logstash/plugin_mixins/opensearch/api_configs.rb

@@ -20,6 +20,18 @@ module LogStash; module PluginMixins; module OpenSearch
         # Password to authenticate to a secure OpenSearch cluster
         :password => { :validate => :password },
 
+        # if auth_type is "aws_iam" then
+        #   Credential resolution logic works as follows :
+        #
+        # - User passed aws_access_key_id and aws_secret_access_key in opensearch configuration
+        # - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+        #   (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET),
+        #   or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK)
+        # - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
+        # - Instance profile credentials delivered through the Amazon EC2 metadata service
+        # - type in auth_type specifies the type of authentication
+        :auth_type => { },
+
         # The document ID for the index. Useful for overwriting existing entries in
         # OpenSearch with the same ID.
         :document_id => { :validate => :string },
@@ -34,6 +46,20 @@ module LogStash; module PluginMixins; module OpenSearch
         # this defaults to a concatenation of the path parameter and "_bulk"
         :bulk_path => { :validate => :string },
 
+        # Maximum number of bytes in bulk requests
+        # The criteria for deciding the default value of target_bulk_bytes is:
+        # 1. We need a number that's less than 10MiB because OpenSearch is commonly
+        #    configured (particular in AWS Opensearch Service) to not accept
+        #    bulks larger than that.
+        # 2. It must be large enough to amortize the connection constant
+        #    across multiple requests.
+        # 3. It must be small enough that even if multiple threads hit this size
+        #    we won't use a lot of heap.
+        :target_bulk_bytes => {
+            :validate => :number,
+            :default => 9 * 1024 * 1024 # 9MiB
+        },
+
         # Pass a set of key value pairs as the URL query string. This query string is added
         # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
         # urls that already have query strings, the one specified here will be appended.

data/lib/logstash/plugin_mixins/opensearch/common.rb

@@ -24,8 +24,8 @@ module LogStash; module PluginMixins; module OpenSearch
     # Perform some OpenSearch options validations and Build the HttpClient.
     # Note that this methods may sets the @user, @password, @hosts and @client ivars as a side effect.
     # @return [HttpClient] the new http client
-    def build_client(distribution_checker=nil)
-      params["distribution_checker"] = distribution_checker
+    def build_client
+
       # the following 3 options validation & setup methods are called inside build_client
       # because they must be executed prior to building the client and logstash
       # monitoring and management rely on directly calling build_client

data/logstash-output-opensearch.gemspec

@@ -1,6 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+signing_key_path = "gem-private_key.pem"
+
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.0.0'
+  s.version         = '1.3.0'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -18,6 +29,11 @@ Gem::Specification.new do |s|
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
+  if $PROGRAM_NAME.end_with?("gem") && ARGV == ["build", __FILE__] && File.exist?(signing_key_path)
+    s.signing_key = signing_key_path
+    s.cert_chain  = ['certs/opensearch-rubygems.pem']
+  end
+
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = {
     "logstash_plugin" => "true",
@@ -25,16 +41,16 @@ Gem::Specification.new do |s|
     "source_code_uri" => "https://github.com/opensearch-project/logstash-output-opensearch"
   }
 
-  s.cert_chain  = ['public.pem']
-  s.signing_key = File.expand_path("private.pem") if $0 =~ /gem\z/
-
   s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
+  s.add_runtime_dependency 'aws-sdk',  '>= 2.11.632', '~> 2'
+  s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
   s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'opensearch-ruby', '~> 1'
 end

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -33,8 +33,8 @@ describe "indexing with http_compression turned on", :integration => true do
   }
   subject { LogStash::Outputs::OpenSearch.new(config) }
 
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
+  let(:opensearch_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{opensearch_url}/#{index}"}
   let(:http_client_options) { {} }
   let(:http_client) do
     Manticore::Client.new(http_client_options)
@@ -49,7 +49,7 @@ describe "indexing with http_compression turned on", :integration => true do
     it "ships events" do
       subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{opensearch_url}/_refresh").call
 
       response = http_client.get("#{index_url}/_count?q=*")
       result = LogStash::Json.load(response.body)
@@ -59,7 +59,13 @@ describe "indexing with http_compression turned on", :integration => true do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end
@@ -67,7 +73,7 @@ describe "indexing with http_compression turned on", :integration => true do
 
   it "sets the correct content-encoding header and body is compressed" do
     expect(subject.client.pool.adapter.client).to receive(:send).
-      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "content-type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
       and_call_original
     subject.multi_receive(events)
   end

data/spec/integration/outputs/create_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "client create actions", :integration => true do
   require "logstash/outputs/opensearch"
 
-  def get_es_output(action, id, version=nil, version_type=nil)
+  def get_output(action, id, version=nil, version_type=nil)
     settings = {
       "manage_template" => true,
       "index" => "logstash-create",
@@ -37,7 +37,7 @@ describe "client create actions", :integration => true do
 
   context "when action => create" do
     it "should create new documents with or without id" do
-      subject = get_es_output("create", "id123")
+      subject = get_output("create", "id123")
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       @client.indices.refresh
@@ -49,27 +49,27 @@ describe "client create actions", :integration => true do
     end
 
     it "should allow default (internal) version" do
-      subject = get_es_output("create", "id123", 43)
+      subject = get_output("create", "id123", 43)
       subject.register
     end
 
     it "should allow internal version" do
-      subject = get_es_output("create", "id123", 43, "internal")
+      subject = get_output("create", "id123", 43, "internal")
       subject.register
     end
 
     it "should not allow external version" do
-      subject = get_es_output("create", "id123", 43, "external")
+      subject = get_output("create", "id123", 43, "external")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gt version" do
-      subject = get_es_output("create", "id123", 43, "external_gt")
+      subject = get_output("create", "id123", 43, "external_gt")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gte version" do
-      subject = get_es_output("create", "id123", 43, "external_gte")
+      subject = get_output("create", "id123", 43, "external_gte")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end

data/spec/integration/outputs/delete_spec.rb

@@ -14,15 +14,15 @@ require "logstash/outputs/opensearch"
 describe "Versioned delete", :integration => true do
   require "logstash/outputs/opensearch"
 
-  let(:es) { get_client }
+  let(:client) { get_client }
 
   before :each do
     # Delete all templates first.
     # Clean ES of data before we start.
-    es.indices.delete_template(:name => "*")
+    client.indices.delete_template(:name => "*")
     # This can fail if there are no indexes, ignore failure.
-    es.indices.delete(:index => "*") rescue nil
-    es.indices.refresh
+    client.indices.delete(:index => "*") rescue nil
+    client.indices.refresh
   end
 
   context "when delete only" do
@@ -48,12 +48,12 @@ describe "Versioned delete", :integration => true do
     it "should ignore non-monotonic external version updates" do
       id = "ev2"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r2 = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r2['_version']).to eq(99)
       expect(r2['_source']['message']).to eq('foo')
     end
@@ -61,12 +61,12 @@ describe "Versioned delete", :integration => true do
     it "should commit monotonic external version updates" do
       id = "ev3"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      expect { client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(OpenSearch::Transport::Transport::Errors::NotFound)
     end
   end
 end

data/spec/integration/outputs/index_spec.rb

@@ -10,8 +10,7 @@
 require_relative "../../../spec/opensearch_spec_helper"
 require "logstash/outputs/opensearch"
 
-describe "TARGET_BULK_BYTES", :integration => true do
-  let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+describe "target_bulk_bytes", :integration => true do
   let(:event_count) { 1000 }
   let(:events) { event_count.times.map { event }.to_a }
   let(:config) {
@@ -32,11 +31,11 @@ describe "TARGET_BULK_BYTES", :integration => true do
   end
 
   describe "batches that are too large for one" do
-    let(:event) { LogStash::Event.new("message" => "a " * (((target_bulk_bytes/2) / event_count)+1)) }
+    let(:event) { LogStash::Event.new("message" => "a " * (((subject.client.target_bulk_bytes/2) / event_count)+1)) }
 
     it "should send in two batches" do
       expect(subject.client).to have_received(:bulk_send).twice do |payload|
-        expect(payload.size).to be <= target_bulk_bytes
+        expect(payload.size).to be <= subject.client.target_bulk_bytes
       end
     end
 
@@ -47,7 +46,7 @@ describe "TARGET_BULK_BYTES", :integration => true do
 
       it "should send in one batch" do
         expect(subject.client).to have_received(:bulk_send).once do |payload|
-          expect(payload.size).to be <= target_bulk_bytes
+          expect(payload.size).to be <= subject.client.target_bulk_bytes
         end
       end
     end
@@ -63,8 +62,8 @@ describe "indexing" do
   let(:events) { event_count.times.map { event }.to_a }
   subject { LogStash::Outputs::OpenSearch.new(config) }
   
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
+  let(:opensearch_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{opensearch_url}/#{index}"}
   let(:http_client_options) { {} }
   let(:http_client) do
     Manticore::Client.new(http_client_options)
@@ -79,7 +78,7 @@ describe "indexing" do
     it "ships events" do
       subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{opensearch_url}/_refresh").call
 
       response = http_client.get("#{index_url}/_count?q=*")
       result = LogStash::Json.load(response.body)
@@ -89,16 +88,22 @@ describe "indexing" do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end
     
     it "sets the correct content-type header" do
-      expected_manticore_opts = {:headers => {"Content-Type" => "application/json"}, :body => anything}
+      expected_manticore_opts = {:headers => {"content-type" => "application/json"}, :body => anything}
       if secure
         expected_manticore_opts = {
-          :headers => {"Content-Type" => "application/json"}, 
+          :headers => {"content-type" => "application/json"},
           :body => anything, 
           :auth => {
             :user => user,
@@ -136,7 +141,7 @@ describe "indexing" do
   describe "a secured indexer", :secure_integration => true do
     let(:user) { "admin" }
     let(:password) { "admin" }
-    let(:es_url) {"https://integration:9200"}
+    let(:opensearch_url) {"https://integration:9200"}
     let(:config) do
       {
         "hosts" => ["integration:9200"],
@@ -161,4 +166,41 @@ describe "indexing" do
     end
     it_behaves_like("an indexer", true)
   end
+
+  describe "a basic auth secured indexer", :secure_integration => true do
+    let(:options) { {
+      :auth_type => {
+        "type"=>"basic",
+        "user" => "admin",
+        "password" => "admin"}
+    } }
+    let(:user) {options[:auth_type]["user"]}
+    let(:password) {options[:auth_type]["password"]}
+    let(:opensearch_url) {"https://integration:9200"}
+    let(:config) do
+      {
+        "hosts" => ["integration:9200"],
+        "auth_type" => {
+          "type"=>"basic",
+          "user" => user,
+          "password" => password},
+        "ssl" => true,
+        "ssl_certificate_verification" => false,
+        "index" => index
+      }
+    end
+    let(:http_client_options) do
+      {
+        :auth => {
+          :user => user,
+          :password => password
+        },
+        :ssl => {
+          :enabled => true,
+          :verify => false
+        }
+      }
+    end
+    it_behaves_like("an indexer", true)
+  end
 end

data/spec/integration/outputs/index_version_spec.rb

@@ -13,15 +13,15 @@ require "logstash/outputs/opensearch"
 describe "Versioned indexing", :integration => true do
   require "logstash/outputs/opensearch"
 
-  let(:es) { get_client }
+  let(:client) { get_client }
 
   before :each do
     # Delete all templates first.
     # Clean OpenSearch of data before we start.
-    es.indices.delete_template(:name => "*")
+    client.indices.delete_template(:name => "*")
     # This can fail if there are no indexes, ignore failure.
-    es.indices.delete(:index => "*") rescue nil
-    es.indices.refresh
+    client.indices.delete(:index => "*") rescue nil
+    client.indices.refresh
   end
 
   context "when index only" do
@@ -46,11 +46,11 @@ describe "Versioned indexing", :integration => true do
 
       it "should default to OpenSearch version" do
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r["_version"]).to eq(1)
         expect(r["_source"]["message"]).to eq('foo')
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r2["_version"]).to eq(2)
         expect(r2["_source"]["message"]).to eq('foobar')
       end
@@ -74,7 +74,7 @@ describe "Versioned indexing", :integration => true do
       it "should respect the external version" do
         id = "ev1"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
       end
@@ -82,12 +82,12 @@ describe "Versioned indexing", :integration => true do
       it "should ignore non-monotonic external version updates" do
         id = "ev2"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(99)
         expect(r2["_source"]["message"]).to eq('foo')
       end
@@ -95,12 +95,12 @@ describe "Versioned indexing", :integration => true do
       it "should commit monotonic external version updates" do
         id = "ev3"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(100)
         expect(r2["_source"]["message"]).to eq('foo')
       end

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -14,7 +14,8 @@ describe "Ingest pipeline execution behavior", :integration => true do
     require "logstash/outputs/opensearch"
     settings = {
       "hosts" => "#{get_host_port()}",
-      "pipeline" => "apache-logs"
+      "pipeline" => "apache-logs",
+      "ecs_compatibility" => "disabled" # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::OpenSearch.new(settings)
   end
@@ -37,8 +38,6 @@ describe "Ingest pipeline execution behavior", :integration => true do
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
-
     # Clean OpenSearch of data before we start.
     @client = get_client
     @client.indices.delete_template(:name => "*")
@@ -50,7 +49,7 @@ describe "Ingest pipeline execution behavior", :integration => true do
     http_client.delete(ingest_url).call
 
     # register pipeline
-    http_client.put(ingest_url, :body => apache_logs_pipeline, :headers => {"Content-Type" => "application/json" }).call
+    http_client.put(ingest_url, :body => apache_logs_pipeline, :headers => {"content-type" => "application/json" }).call
 
     #TODO: Use esclient
     #@client.ingest.put_pipeline :id => 'apache_pipeline', :body => pipeline_defintion

data/spec/integration/outputs/metrics_spec.rb

@@ -24,8 +24,6 @@ describe "metrics", :integration => true do
   let(:document_level_metrics) { subject.instance_variable_get(:@document_level_metrics) }
 
   before :each do
-    require "elasticsearch"
-
     # Clean OpenSearch of data before we start.
     @client = get_client
     clean(@client)

data/spec/integration/outputs/no_opensearch_on_startup_spec.rb

@@ -27,7 +27,6 @@ describe "opensearch is down on startup", :integration => true do
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
     allow(Stud).to receive(:stoppable_sleep)
 
     # Clean OpenSearch of data before we start.

data/spec/integration/outputs/painless_update_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
   require "logstash/outputs/opensearch"
 
-  def get_es_output( options={} )
+  def get_output( options={} )
     settings = {
       "manage_template" => true,
       "index" => "logstash-update",
@@ -42,7 +42,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
   context "scripted updates" do
 
     it "should increment a counter with event/doc 'count' variable with inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "123",
         'script' => 'ctx._source.counter += params.event.counter',
         'script_type' => 'inline'
@@ -54,7 +54,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "123",
         'doc_as_upsert' => true,
         'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
@@ -67,7 +67,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "456",
         'doc_as_upsert' => true,
         'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
@@ -90,7 +90,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
         plugin_parameters.merge!('script_lang' => '')
 
-        subject = get_es_output(plugin_parameters)
+        subject = get_output(plugin_parameters)
         subject.register
         subject.multi_receive([LogStash::Event.new("count" => 4 )])
         r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -101,7 +101,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
   context "when update with upsert" do
     it "should create new documents with provided upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject = get_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -109,7 +109,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should create new documents with event/doc as upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -117,7 +117,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should fail on documents with event/doc as upsert at external version" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end
@@ -126,7 +126,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
     context 'with an inline script' do
       it "should create new documents with upsert content" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
+        subject = get_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
         subject.register
 
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
@@ -135,7 +135,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
       end
 
       it "should create new documents with event/doc as script params" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
+        subject = get_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 1)])
         @client.indices.refresh

data/spec/integration/outputs/parent_spec.rb

@@ -10,7 +10,7 @@
 require_relative "../../../spec/opensearch_spec_helper"
 require "logstash/outputs/opensearch"
 
-context "join field tests", :integration => true do
+context "join field tests", :integration => true && OpenSearchHelper.check_version?("<2") do
 
   shared_examples "a join field based parent indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
@@ -25,7 +25,7 @@ context "join field tests", :integration => true do
     let(:parent_relation) { "parent_type" }
     let(:child_relation) { "child_type" }
     let(:default_headers) {
-      {"Content-Type" => "application/json"}
+      {"content-type" => "application/json"}
     }
     subject { LogStash::Outputs::OpenSearch.new(config) }