logstash-output-opensearch 1.3.0-java → 2.0.3-java

data/lib/logstash/outputs/opensearch.rb

@@ -199,6 +199,13 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
   # here like `pipeline => "%{INGEST_PIPELINE}"`
   config :pipeline, :validate => :string, :default => nil
+  
+  # When set to true, use legacy templates via the _template API
+  # When false, use index templates using the _index_template API
+  config :legacy_template, :validate => :boolean, :default => true
+
+  # The OpenSearch server major version to use when it's not available from the Healthcheck endpoint.
+  config :default_server_major_version, :validate => :number
 
   attr_reader :client
   attr_reader :default_index

data/lib/logstash/plugin_mixins/opensearch/common.rb

@@ -87,6 +87,7 @@ module LogStash; module PluginMixins; module OpenSearch
 
     def discover_cluster_uuid
       return unless defined?(plugin_metadata)
+      return if params && params['auth_type'] && params['auth_type']['service_name'] == "aoss" # AOSS doesn't support GET /
       cluster_info = client.get('/')
       plugin_metadata.set(:cluster_uuid, cluster_info['cluster_uuid'])
     rescue => e

data/logstash-output-opensearch.gemspec

@@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem"
 
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.3.0'
+  s.version         = '2.0.3'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
-  s.add_runtime_dependency 'aws-sdk',  '>= 2.11.632', '~> 2'
+  s.add_runtime_dependency 'aws-sdk', '~> 3'
   s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'

data/spec/integration/outputs/index_spec.rb

@@ -140,7 +140,7 @@ describe "indexing" do
   end
   describe "a secured indexer", :secure_integration => true do
     let(:user) { "admin" }
-    let(:password) { "admin" }
+    let(:password) { OpenSearchHelper.admin_password }
     let(:opensearch_url) {"https://integration:9200"}
     let(:config) do
       {
@@ -172,7 +172,8 @@ describe "indexing" do
       :auth_type => {
         "type"=>"basic",
         "user" => "admin",
-        "password" => "admin"}
+        "password" => OpenSearchHelper.admin_password
+        }
     } }
     let(:user) {options[:auth_type]["user"]}
     let(:password) {options[:auth_type]["password"]}

data/spec/opensearch_spec_helper.rb

@@ -72,6 +72,15 @@ module OpenSearchHelper
     end
   end
 
+  # set admin password based on version
+  def self.admin_password
+    if check_version?('< 2.12.0') || check_version?('> 7')
+      "admin"
+    else
+      "myStrongPassword123!"
+    end
+  end
+
   def clean(client)
     client.indices.delete_template(:name => "*")
     client.indices.delete_index_template(:name => "logstash*") rescue nil

data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb

@@ -61,31 +61,89 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
         "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
         "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
     } }
+    let(:options_svc) { {
+      :auth_type => {
+        "type"=>"aws_iam",
+        "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
+        "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        "service_name"=>"svc_test"}
+    } }
     subject { described_class.new(logger, options) }
     let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
-    let(:sign_aws_request) {  }
 
-    it "should validate AWS IAM credentials initialization" do
-      expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
-    end
+    let(:expected_uri) {
+      expected_uri = uri.clone
+      expected_uri.path = "/"
+      expected_uri
+    }
 
-    it "should validate signing aws request" do
+    let(:resp) {
       resp = double("response")
       allow(resp).to receive(:call)
       allow(resp).to receive(:code).and_return(200)
-      allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+      resp
+    }
 
-      expected_uri = uri.clone
-      expected_uri.path = "/"
+    context 'with a signer' do
+      let(:sign_aws_request) {  }
 
-      expect(subject.manticore).to receive(:get).
-        with(expected_uri.to_s, {
-          :headers => {"content-type"=> "application/json"}
-        }
+      it "should validate AWS IAM credentials initialization" do
+        expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
+        expect(subject.get_service_name).to eq("es")
+      end
+
+      it "should validate AWS IAM service_name config" do
+        expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
+        expect(subject.get_service_name).to eq("svc_test")
+      end
+
+      it "should validate signing aws request" do
+        allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+
+        expect(subject.manticore).to receive(:get).
+          with(expected_uri.to_s, {
+            :headers => {"content-type"=> "application/json"}
+          }
+          ).and_return resp
+
+        expect(subject).to receive(:sign_aws_request)
+        subject.perform_request(uri, :get, "/")
+      end
+    end
+
+    context 'sign_aws_request' do
+      it 'handles UTF-8' do
+        encoded_body = body = "boîte de réception"
+        expect_any_instance_of(Aws::Sigv4::Signer).to receive(:sign_request).with(hash_including({
+            body: body,
+        })).and_return(
+          double(headers: {})
+        )
+        expect(subject.manticore).to receive(:post).
+          with(expected_uri.to_s, {
+            :body => encoded_body,
+            :headers => {"content-type"=> "application/json"}
+          }
         ).and_return resp
+        subject.perform_request(uri, :post, "/", { body: encoded_body })
+      end
 
-      expect(subject).to receive(:sign_aws_request)
-      subject.perform_request(uri, :get, "/")
+      it 'encodes body before signing to match manticore adapter encoding' do
+        body = "boîte de réception"
+        encoded_body = body.encode("ISO-8859-1")
+        expect_any_instance_of(Aws::Sigv4::Signer).to receive(:sign_request).with(hash_including({
+            body: body,
+        })).and_return(
+          double(headers: {})
+        )
+        expect(subject.manticore).to receive(:post).
+          with(expected_uri.to_s, {
+            :body => encoded_body,
+            :headers => {"content-type"=> "application/json"}
+          }
+        ).and_return resp
+        subject.perform_request(uri, :post, "/", { body: encoded_body })
+      end
     end
   end
 

data/spec/unit/outputs/opensearch/http_client_spec.rb

@@ -162,6 +162,26 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
     end
   end
 
+  describe "legacy_template" do
+    let(:template_name) { "logstash" }
+    let(:template) { {} }
+    let(:get_response) {
+      double("response", :body => {}, :code => 404)
+    }
+    [true, false].each do |legacy_template|
+      context "when legacy_template => #{legacy_template}" do
+        let(:base_options) { super().merge(:client_settings => {:legacy_template => legacy_template}) }
+        subject { described_class.new(base_options) }
+        endpoint = legacy_template ? "_template" : "_index_template"
+        it "should call template_endpoint #{endpoint}" do
+          expect(subject.pool).to receive(:head).with("/#{endpoint}/#{template_name}").and_return(get_response)
+          expect(subject.pool).to receive(:put).with("/#{endpoint}/#{template_name}", nil, anything).and_return(get_response)  
+          subject.template_install(template_name, template)
+        end
+      end
+    end
+  end
+
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

data/spec/unit/outputs/opensearch/template_manager_spec.rb

@@ -13,27 +13,15 @@ require "logstash/outputs/opensearch/template_manager"
 describe LogStash::Outputs::OpenSearch::TemplateManager do
 
   describe ".default_template_path" do
-    [1, 2].each do |major_version|
-      context "when ECS is disabled with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
-        end
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
-        end
-      end
-    end
     [7, 1, 2].each do |major_version|
-      context "when ECS v1 is requested with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :v1)).to end_with("/templates/ecs-v1/#{major_version}x.json")
-        end
-      end
-    end
-    [1, 2].each do |major_version|
-      context "when ECS v8 is requested with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
+      [:disabled, :v1, :v8].each do |ecs_ver|
+        [true, false].each do |legacy_template|
+          context "when ECS is #{ecs_ver} with OpenSearch #{major_version}.x legacy_template:#{legacy_template}" do
+            suffix = legacy_template ? "" : "_index"
+            it 'resolves' do
+              expect(described_class.default_template_path(major_version, ecs_ver, legacy_template)).to end_with("/templates/ecs-#{ecs_ver}/#{major_version}x#{suffix}.json")
+            end
+          end
         end
       end
     end

data/spec/unit/outputs/opensearch_spec.rb

@@ -776,6 +776,30 @@ describe LogStash::Outputs::OpenSearch do
       expect(logger).to have_received(:error).with(/Unable to retrieve OpenSearch cluster uuid/i, anything)
     end
 
+    context 'with iam auth' do
+      context 'es' do
+        let(:options) { { 'hosts' => '127.0.0.1:9999', 'auth_type' => { 'service_name' => 'es' } } }
+        it "logs inability to retrieve uuid" do
+          allow(subject).to receive(:install_template)
+          subject.register
+          subject.send :wait_for_successful_connection
+    
+          expect(logger).to have_received(:error).with(/Unable to retrieve OpenSearch cluster uuid/i, anything)
+        end
+      end
+
+      context 'aoss' do
+        let(:options) { { 'hosts' => '127.0.0.1:9999', 'auth_type' => { 'service_name' => 'aoss' } } }
+        it "does not attempt to retrieve uuid" do
+          allow(subject).to receive(:install_template)
+          subject.register
+          subject.send :wait_for_successful_connection
+
+          expect(logger).to_not have_received(:error)
+        end
+      end
+    end
+
     it "logs template install failure" do
       allow(subject).to receive(:discover_cluster_uuid)
       subject.register

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 2.0.3
 platform: java
 authors:
 - Elastic
@@ -11,9 +11,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDfDCCAmSgAwIBAgIBATANBgkqhkiG9w0BAQUFADBCMRMwEQYDVQQDDApvcGVu
+  MIIDfDCCAmSgAwIBAgIBATANBgkqhkiG9w0BAQsFADBCMRMwEQYDVQQDDApvcGVu
   c2VhcmNoMRYwFAYKCZImiZPyLGQBGRYGYW1hem9uMRMwEQYKCZImiZPyLGQBGRYD
-  Y29tMB4XDTIyMDgxNzE3NTIzNFoXDTIzMDgxNzE3NTIzNFowQjETMBEGA1UEAwwK
+  Y29tMB4XDTI0MDcyNjIzMjIyN1oXDTI1MDcyNjIzMjIyN1owQjETMBEGA1UEAwwK
   b3BlbnNlYXJjaDEWMBQGCgmSJomT8ixkARkWBmFtYXpvbjETMBEGCgmSJomT8ixk
   ARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1z3/jcitjV
   umXwRFn+JSBBd36qZB54Dtucf6+E2fmNPzBRhgYN5XJy/+clQJ9NIJV7C8H52P3V
@@ -24,14 +24,14 @@ cert_chain:
   zfR37/NQFkECAwEAAaN9MHswCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0O
   BBYEFJJ2myhLXK742btavNbG0IWrMNBIMCAGA1UdEQQZMBeBFW9wZW5zZWFyY2hA
   YW1hem9uLmNvbTAgBgNVHRIEGTAXgRVvcGVuc2VhcmNoQGFtYXpvbi5jb20wDQYJ
-  KoZIhvcNAQEFBQADggEBAH5pWLYwKWFh1OjdCReGz/VEAiF4jXXputoN5Z8ga+1Z
-  lg8/diJf0PlP2B46PdmxH/TVc/o+qglNO2cHVEp8xZfEd83dfioOBeK90URQUpC5
-  UZmO0LZusg46SQKwKa2ukpIy2fNi3PeHRiV+W2Zv69GoWppyLun+fMez7wVoah2r
-  r5ROUYuAvFUvga1Vm+49pKiPM5n+MAYP5t/vWhgymY3SYQ1TfewkvKAFiFXikOR+
-  r+j7FLyKuk5DzIxiCp8QN5dU71BbGUmsHf/C5UV76WLPOFX/szeaHhPwpjR3sK7r
-  5zLgCV1KP7cgDdCYMlmZGeSViU8NV+Yy8/ghrzGpqVw=
+  KoZIhvcNAQELBQADggEBAB1D6Ba88KkTApeUl0Iv/WKie1WNi+6o3KQhqqLt+xjB
+  oiBLLdQpKwH7k/TkCmfo9/8lY1sa3Pxckuw2fNVDaVhEHX56fmTUPoOjkyPS4H19
+  YEKFBeY6U1aF7a2piN58j4EcPP7Kv0KD3RlaEwKbYJTiOy/0f5XjWqVYKOS+DrjM
+  EMhX0gWR5oucydDNSi5hkggPYqCZnW9q6yo+k/FW+DK33CeUm69D6elFx7qPGhEx
+  m0rB25J1sO2yjlalRdBoV5p7OKaWG4CGMr+q6vCC2za7yrtZiYOOv+KpWmheTwds
+  P1nbdIL2eAXGfNU3cbtYisS1sSYi7VSkS52pVk8Jgtw=
   -----END CERTIFICATE-----
-date: 2022-08-17 00:00:00.000000000 Z
+date: 2024-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -43,8 +43,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.0.0
   name: manticore
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -63,8 +63,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.0'
   name: stud
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -83,8 +83,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -100,8 +100,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
   name: logstash-mixin-ecs_compatibility_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -110,23 +110,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.11.632
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '3'
   name: aws-sdk
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.11.632
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -137,8 +131,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
   name: json
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -154,8 +148,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -168,8 +162,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -182,8 +176,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: flores
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -196,8 +190,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0.6'
   name: cabin
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -210,8 +204,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
   name: opensearch-ruby
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -237,6 +231,8 @@ files:
 - README.md
 - RELEASING.md
 - SECURITY.md
+- docs/building_custom_docker_images.md
+- docs/ecs_compatibility.md
 - lib/logstash/outputs/opensearch.rb
 - lib/logstash/outputs/opensearch/http_client.rb
 - lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb
@@ -244,11 +240,17 @@ files:
 - lib/logstash/outputs/opensearch/http_client_builder.rb
 - lib/logstash/outputs/opensearch/template_manager.rb
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json
 - lib/logstash/plugin_mixins/opensearch/api_configs.rb
 - lib/logstash/plugin_mixins/opensearch/common.rb
 - logstash-output-opensearch.gemspec
@@ -306,7 +308,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.20
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Stores logs in OpenSearch