logstash-output-opensearch 1.1.0-java → 2.0.0-java

data/lib/logstash/outputs/opensearch.rb

@@ -78,7 +78,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   include(LogStash::PluginMixins::OpenSearch::Common)
 
   # ecs_compatibility option, provided by Logstash core or the support adapter.
-  include(LogStash::PluginMixins::ECSCompatibilitySupport)
+  include(LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8))
 
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::OpenSearch::APIConfigs)
@@ -95,7 +95,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
   #   would use the foo field for the action
   #
-  # For more details on actions, check out the https://docs-beta.opensearch.org/opensearch/rest-api/bulk/[OpenSearch bulk API documentation]
+  # For more details on actions, check out the https://opensearch.org/docs/opensearch/rest-api/document-apis/bulk/[OpenSearch bulk API documentation]
   config :action, :validate => :string, :default => "index"
 
   # The index to write events to. This can be dynamic using the `%{foo}` syntax.
@@ -199,6 +199,13 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
   # here like `pipeline => "%{INGEST_PIPELINE}"`
   config :pipeline, :validate => :string, :default => nil
+  
+  # When set to true, use legacy templates via the _template API
+  # When false, use index templates using the _index_template API
+  config :legacy_template, :validate => :boolean, :default => true
+
+  # The OpenSearch server major version to use when it's not available from the Healthcheck endpoint.
+  config :default_server_major_version, :validate => :number
 
   attr_reader :client
   attr_reader :default_index
@@ -419,7 +426,7 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
     when :disabled
       @default_index = "logstash-%{+yyyy.MM.dd}"
       @default_template_name = 'logstash'
-    when :v1
+    when :v1, :v8
       @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
       @default_template_name = 'ecs-logstash'
     else

data/lib/logstash/plugin_mixins/opensearch/api_configs.rb

@@ -20,6 +20,18 @@ module LogStash; module PluginMixins; module OpenSearch
         # Password to authenticate to a secure OpenSearch cluster
         :password => { :validate => :password },
 
+        # if auth_type is "aws_iam" then
+        #   Credential resolution logic works as follows :
+        #
+        # - User passed aws_access_key_id and aws_secret_access_key in opensearch configuration
+        # - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+        #   (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET),
+        #   or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK)
+        # - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
+        # - Instance profile credentials delivered through the Amazon EC2 metadata service
+        # - type in auth_type specifies the type of authentication
+        :auth_type => { },
+
         # The document ID for the index. Useful for overwriting existing entries in
         # OpenSearch with the same ID.
         :document_id => { :validate => :string },
@@ -34,6 +46,20 @@ module LogStash; module PluginMixins; module OpenSearch
         # this defaults to a concatenation of the path parameter and "_bulk"
         :bulk_path => { :validate => :string },
 
+        # Maximum number of bytes in bulk requests
+        # The criteria for deciding the default value of target_bulk_bytes is:
+        # 1. We need a number that's less than 10MiB because OpenSearch is commonly
+        #    configured (particular in AWS Opensearch Service) to not accept
+        #    bulks larger than that.
+        # 2. It must be large enough to amortize the connection constant
+        #    across multiple requests.
+        # 3. It must be small enough that even if multiple threads hit this size
+        #    we won't use a lot of heap.
+        :target_bulk_bytes => {
+            :validate => :number,
+            :default => 9 * 1024 * 1024 # 9MiB
+        },
+
         # Pass a set of key value pairs as the URL query string. This query string is added
         # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
         # urls that already have query strings, the one specified here will be appended.

data/logstash-output-opensearch.gemspec

@@ -1,6 +1,17 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+signing_key_path = "gem-private_key.pem"
+
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.1.0'
+  s.version         = '2.0.0'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -18,8 +29,10 @@ Gem::Specification.new do |s|
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
-  s.cert_chain  = ['certs/opensearch-rubygems.pem']
-  s.signing_key = File.expand_path("private_key.pem") if $0 =~ /gem\z/
+  if $PROGRAM_NAME.end_with?("gem") && ARGV == ["build", __FILE__] && File.exist?(signing_key_path)
+    s.signing_key = signing_key_path
+    s.cert_chain  = ['certs/opensearch-rubygems.pem']
+  end
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = {
@@ -32,9 +45,12 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
+  s.add_runtime_dependency 'aws-sdk', '~> 3'
+  s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
   s.add_development_dependency 'cabin', ['~> 0.6']
+  s.add_development_dependency 'opensearch-ruby', '~> 1'
 end

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -33,8 +33,8 @@ describe "indexing with http_compression turned on", :integration => true do
   }
   subject { LogStash::Outputs::OpenSearch.new(config) }
 
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
+  let(:opensearch_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{opensearch_url}/#{index}"}
   let(:http_client_options) { {} }
   let(:http_client) do
     Manticore::Client.new(http_client_options)
@@ -49,7 +49,7 @@ describe "indexing with http_compression turned on", :integration => true do
     it "ships events" do
       subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{opensearch_url}/_refresh").call
 
       response = http_client.get("#{index_url}/_count?q=*")
       result = LogStash::Json.load(response.body)
@@ -59,7 +59,13 @@ describe "indexing with http_compression turned on", :integration => true do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end

data/spec/integration/outputs/create_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "client create actions", :integration => true do
   require "logstash/outputs/opensearch"
 
-  def get_es_output(action, id, version=nil, version_type=nil)
+  def get_output(action, id, version=nil, version_type=nil)
     settings = {
       "manage_template" => true,
       "index" => "logstash-create",
@@ -37,7 +37,7 @@ describe "client create actions", :integration => true do
 
   context "when action => create" do
     it "should create new documents with or without id" do
-      subject = get_es_output("create", "id123")
+      subject = get_output("create", "id123")
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       @client.indices.refresh
@@ -49,27 +49,27 @@ describe "client create actions", :integration => true do
     end
 
     it "should allow default (internal) version" do
-      subject = get_es_output("create", "id123", 43)
+      subject = get_output("create", "id123", 43)
       subject.register
     end
 
     it "should allow internal version" do
-      subject = get_es_output("create", "id123", 43, "internal")
+      subject = get_output("create", "id123", 43, "internal")
       subject.register
     end
 
     it "should not allow external version" do
-      subject = get_es_output("create", "id123", 43, "external")
+      subject = get_output("create", "id123", 43, "external")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gt version" do
-      subject = get_es_output("create", "id123", 43, "external_gt")
+      subject = get_output("create", "id123", 43, "external_gt")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gte version" do
-      subject = get_es_output("create", "id123", 43, "external_gte")
+      subject = get_output("create", "id123", 43, "external_gte")
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end

data/spec/integration/outputs/delete_spec.rb

@@ -14,15 +14,15 @@ require "logstash/outputs/opensearch"
 describe "Versioned delete", :integration => true do
   require "logstash/outputs/opensearch"
 
-  let(:es) { get_client }
+  let(:client) { get_client }
 
   before :each do
     # Delete all templates first.
     # Clean ES of data before we start.
-    es.indices.delete_template(:name => "*")
+    client.indices.delete_template(:name => "*")
     # This can fail if there are no indexes, ignore failure.
-    es.indices.delete(:index => "*") rescue nil
-    es.indices.refresh
+    client.indices.delete(:index => "*") rescue nil
+    client.indices.refresh
   end
 
   context "when delete only" do
@@ -48,12 +48,12 @@ describe "Versioned delete", :integration => true do
     it "should ignore non-monotonic external version updates" do
       id = "ev2"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-      r2 = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r2 = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r2['_version']).to eq(99)
       expect(r2['_source']['message']).to eq('foo')
     end
@@ -61,12 +61,12 @@ describe "Versioned delete", :integration => true do
     it "should commit monotonic external version updates" do
       id = "ev3"
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-      r = es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
+      r = client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true)
       expect(r['_version']).to eq(99)
       expect(r['_source']['message']).to eq('foo')
 
       subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-      expect { es.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      expect { client.get(:index => 'logstash-delete', :type => doc_type, :id => id, :refresh => true) }.to raise_error(OpenSearch::Transport::Transport::Errors::NotFound)
     end
   end
 end

data/spec/integration/outputs/index_spec.rb

@@ -10,8 +10,7 @@
 require_relative "../../../spec/opensearch_spec_helper"
 require "logstash/outputs/opensearch"
 
-describe "TARGET_BULK_BYTES", :integration => true do
-  let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+describe "target_bulk_bytes", :integration => true do
   let(:event_count) { 1000 }
   let(:events) { event_count.times.map { event }.to_a }
   let(:config) {
@@ -32,11 +31,11 @@ describe "TARGET_BULK_BYTES", :integration => true do
   end
 
   describe "batches that are too large for one" do
-    let(:event) { LogStash::Event.new("message" => "a " * (((target_bulk_bytes/2) / event_count)+1)) }
+    let(:event) { LogStash::Event.new("message" => "a " * (((subject.client.target_bulk_bytes/2) / event_count)+1)) }
 
     it "should send in two batches" do
       expect(subject.client).to have_received(:bulk_send).twice do |payload|
-        expect(payload.size).to be <= target_bulk_bytes
+        expect(payload.size).to be <= subject.client.target_bulk_bytes
       end
     end
 
@@ -47,7 +46,7 @@ describe "TARGET_BULK_BYTES", :integration => true do
 
       it "should send in one batch" do
         expect(subject.client).to have_received(:bulk_send).once do |payload|
-          expect(payload.size).to be <= target_bulk_bytes
+          expect(payload.size).to be <= subject.client.target_bulk_bytes
         end
       end
     end
@@ -63,8 +62,8 @@ describe "indexing" do
   let(:events) { event_count.times.map { event }.to_a }
   subject { LogStash::Outputs::OpenSearch.new(config) }
   
-  let(:es_url) { "http://#{get_host_port}" }
-  let(:index_url) {"#{es_url}/#{index}"}
+  let(:opensearch_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{opensearch_url}/#{index}"}
   let(:http_client_options) { {} }
   let(:http_client) do
     Manticore::Client.new(http_client_options)
@@ -79,7 +78,7 @@ describe "indexing" do
     it "ships events" do
       subject.multi_receive(events)
 
-      http_client.post("#{es_url}/_refresh").call
+      http_client.post("#{opensearch_url}/_refresh").call
 
       response = http_client.get("#{index_url}/_count?q=*")
       result = LogStash::Json.load(response.body)
@@ -89,7 +88,13 @@ describe "indexing" do
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end
@@ -136,7 +141,7 @@ describe "indexing" do
   describe "a secured indexer", :secure_integration => true do
     let(:user) { "admin" }
     let(:password) { "admin" }
-    let(:es_url) {"https://integration:9200"}
+    let(:opensearch_url) {"https://integration:9200"}
     let(:config) do
       {
         "hosts" => ["integration:9200"],
@@ -161,4 +166,41 @@ describe "indexing" do
     end
     it_behaves_like("an indexer", true)
   end
+
+  describe "a basic auth secured indexer", :secure_integration => true do
+    let(:options) { {
+      :auth_type => {
+        "type"=>"basic",
+        "user" => "admin",
+        "password" => "admin"}
+    } }
+    let(:user) {options[:auth_type]["user"]}
+    let(:password) {options[:auth_type]["password"]}
+    let(:opensearch_url) {"https://integration:9200"}
+    let(:config) do
+      {
+        "hosts" => ["integration:9200"],
+        "auth_type" => {
+          "type"=>"basic",
+          "user" => user,
+          "password" => password},
+        "ssl" => true,
+        "ssl_certificate_verification" => false,
+        "index" => index
+      }
+    end
+    let(:http_client_options) do
+      {
+        :auth => {
+          :user => user,
+          :password => password
+        },
+        :ssl => {
+          :enabled => true,
+          :verify => false
+        }
+      }
+    end
+    it_behaves_like("an indexer", true)
+  end
 end

data/spec/integration/outputs/index_version_spec.rb

@@ -13,15 +13,15 @@ require "logstash/outputs/opensearch"
 describe "Versioned indexing", :integration => true do
   require "logstash/outputs/opensearch"
 
-  let(:es) { get_client }
+  let(:client) { get_client }
 
   before :each do
     # Delete all templates first.
     # Clean OpenSearch of data before we start.
-    es.indices.delete_template(:name => "*")
+    client.indices.delete_template(:name => "*")
     # This can fail if there are no indexes, ignore failure.
-    es.indices.delete(:index => "*") rescue nil
-    es.indices.refresh
+    client.indices.delete(:index => "*") rescue nil
+    client.indices.refresh
   end
 
   context "when index only" do
@@ -46,11 +46,11 @@ describe "Versioned indexing", :integration => true do
 
       it "should default to OpenSearch version" do
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r["_version"]).to eq(1)
         expect(r["_source"]["message"]).to eq('foo')
         subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => "123", :refresh => true)
         expect(r2["_version"]).to eq(2)
         expect(r2["_source"]["message"]).to eq('foobar')
       end
@@ -74,7 +74,7 @@ describe "Versioned indexing", :integration => true do
       it "should respect the external version" do
         id = "ev1"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
       end
@@ -82,12 +82,12 @@ describe "Versioned indexing", :integration => true do
       it "should ignore non-monotonic external version updates" do
         id = "ev2"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(99)
         expect(r2["_source"]["message"]).to eq('foo')
       end
@@ -95,12 +95,12 @@ describe "Versioned indexing", :integration => true do
       it "should commit monotonic external version updates" do
         id = "ev3"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-        r = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r["_version"]).to eq(99)
         expect(r["_source"]["message"]).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
-        r2 = es.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
+        r2 = client.get(:index => 'logstash-index', :type => doc_type, :id => id, :refresh => true)
         expect(r2["_version"]).to eq(100)
         expect(r2["_source"]["message"]).to eq('foo')
       end

data/spec/integration/outputs/ingest_pipeline_spec.rb

@@ -14,7 +14,8 @@ describe "Ingest pipeline execution behavior", :integration => true do
     require "logstash/outputs/opensearch"
     settings = {
       "hosts" => "#{get_host_port()}",
-      "pipeline" => "apache-logs"
+      "pipeline" => "apache-logs",
+      "ecs_compatibility" => "disabled" # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::OpenSearch.new(settings)
   end
@@ -37,8 +38,6 @@ describe "Ingest pipeline execution behavior", :integration => true do
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
-
     # Clean OpenSearch of data before we start.
     @client = get_client
     @client.indices.delete_template(:name => "*")

data/spec/integration/outputs/metrics_spec.rb

@@ -24,8 +24,6 @@ describe "metrics", :integration => true do
   let(:document_level_metrics) { subject.instance_variable_get(:@document_level_metrics) }
 
   before :each do
-    require "elasticsearch"
-
     # Clean OpenSearch of data before we start.
     @client = get_client
     clean(@client)

data/spec/integration/outputs/no_opensearch_on_startup_spec.rb

@@ -27,7 +27,6 @@ describe "opensearch is down on startup", :integration => true do
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
     allow(Stud).to receive(:stoppable_sleep)
 
     # Clean OpenSearch of data before we start.

data/spec/integration/outputs/painless_update_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless' do
   require "logstash/outputs/opensearch"
 
-  def get_es_output( options={} )
+  def get_output( options={} )
     settings = {
       "manage_template" => true,
       "index" => "logstash-update",
@@ -42,7 +42,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
   context "scripted updates" do
 
     it "should increment a counter with event/doc 'count' variable with inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "123",
         'script' => 'ctx._source.counter += params.event.counter',
         'script_type' => 'inline'
@@ -54,7 +54,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "123",
         'doc_as_upsert' => true,
         'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
@@ -67,7 +67,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
+      subject = get_output({
         'document_id' => "456",
         'doc_as_upsert' => true,
         'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
@@ -90,7 +90,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
         plugin_parameters.merge!('script_lang' => '')
 
-        subject = get_es_output(plugin_parameters)
+        subject = get_output(plugin_parameters)
         subject.register
         subject.multi_receive([LogStash::Event.new("count" => 4 )])
         r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -101,7 +101,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
   context "when update with upsert" do
     it "should create new documents with provided upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject = get_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -109,7 +109,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should create new documents with event/doc as upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -117,7 +117,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
     end
 
     it "should fail on documents with event/doc as upsert at external version" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end
@@ -126,7 +126,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
 
     context 'with an inline script' do
       it "should create new documents with upsert content" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
+        subject = get_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
         subject.register
 
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
@@ -135,7 +135,7 @@ describe "Update actions using painless scripts", :integration => true, :update_
       end
 
       it "should create new documents with event/doc as script params" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
+        subject = get_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'scripted_upsert' => true, 'script_type' => 'inline' })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 1)])
         @client.indices.refresh

data/spec/integration/outputs/parent_spec.rb

@@ -10,7 +10,7 @@
 require_relative "../../../spec/opensearch_spec_helper"
 require "logstash/outputs/opensearch"
 
-context "join field tests", :integration => true do
+context "join field tests", :integration => true && OpenSearchHelper.check_version?("<2") do
 
   shared_examples "a join field based parent indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }

data/spec/integration/outputs/retry_spec.rb

@@ -46,14 +46,14 @@ describe "failures in bulk class expected behavior", :integration => true do
       "template_overwrite" => true,
       "hosts" => get_host_port(),
       "retry_max_interval" => 64,
-      "retry_initial_interval" => 2
+      "retry_initial_interval" => 2,
+      "ecs_compatibility" => "disabled" # specs are tightly tied to non-ECS defaults
     }
     next LogStash::Outputs::OpenSearch.new(settings)
   end
 
   before :each do
     # Delete all templates first.
-    require "elasticsearch"
     allow(Stud).to receive(:stoppable_sleep)
 
     # Clean OpenSearch of data before we start.

data/spec/integration/outputs/sniffer_spec.rb

@@ -15,9 +15,9 @@ require "socket"
 describe "pool sniffer", :integration => true do
   let(:logger) { Cabin::Channel.get }
   let(:adapter) { LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter.new(logger) }
-  let(:es_host) { get_host_port.split(":").first }
+  let(:opensearch_host) { get_host_port.split(":").first }
   let(:es_port) { get_host_port.split(":").last }
-  let(:es_ip) { IPSocket.getaddress(es_host) }
+  let(:opensearch_ip) { IPSocket.getaddress(opensearch_host) }
   let(:initial_urls) { [::LogStash::Util::SafeURI.new("http://#{get_host_port}")] }
   let(:options) do
     {