logstash-output-opensearch 1.3.0-java → 2.0.2-java

data/lib/logstash/outputs/opensearch.rb

@@ -199,6 +199,13 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
   # here like `pipeline => "%{INGEST_PIPELINE}"`
   config :pipeline, :validate => :string, :default => nil
+  
+  # When set to true, use legacy templates via the _template API
+  # When false, use index templates using the _index_template API
+  config :legacy_template, :validate => :boolean, :default => true
+
+  # The OpenSearch server major version to use when it's not available from the Healthcheck endpoint.
+  config :default_server_major_version, :validate => :number
 
   attr_reader :client
   attr_reader :default_index

data/logstash-output-opensearch.gemspec

@@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem"
 
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.3.0'
+  s.version         = '2.0.2'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
-  s.add_runtime_dependency 'aws-sdk',  '>= 2.11.632', '~> 2'
+  s.add_runtime_dependency 'aws-sdk', '~> 3'
   s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'

data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb

@@ -61,31 +61,89 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
         "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
         "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
     } }
+    let(:options_svc) { {
+      :auth_type => {
+        "type"=>"aws_iam",
+        "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
+        "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        "service_name"=>"svc_test"}
+    } }
     subject { described_class.new(logger, options) }
     let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
-    let(:sign_aws_request) {  }
 
-    it "should validate AWS IAM credentials initialization" do
-      expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
-    end
+    let(:expected_uri) {
+      expected_uri = uri.clone
+      expected_uri.path = "/"
+      expected_uri
+    }
 
-    it "should validate signing aws request" do
+    let(:resp) {
       resp = double("response")
       allow(resp).to receive(:call)
       allow(resp).to receive(:code).and_return(200)
-      allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+      resp
+    }
 
-      expected_uri = uri.clone
-      expected_uri.path = "/"
+    context 'with a signer' do
+      let(:sign_aws_request) {  }
 
-      expect(subject.manticore).to receive(:get).
-        with(expected_uri.to_s, {
-          :headers => {"content-type"=> "application/json"}
-        }
+      it "should validate AWS IAM credentials initialization" do
+        expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
+        expect(subject.get_service_name).to eq("es")
+      end
+
+      it "should validate AWS IAM service_name config" do
+        expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
+        expect(subject.get_service_name).to eq("svc_test")
+      end
+
+      it "should validate signing aws request" do
+        allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+
+        expect(subject.manticore).to receive(:get).
+          with(expected_uri.to_s, {
+            :headers => {"content-type"=> "application/json"}
+          }
+          ).and_return resp
+
+        expect(subject).to receive(:sign_aws_request)
+        subject.perform_request(uri, :get, "/")
+      end
+    end
+
+    context 'sign_aws_request' do
+      it 'handles UTF-8' do
+        encoded_body = body = "boîte de réception"
+        expect_any_instance_of(Aws::Sigv4::Signer).to receive(:sign_request).with(hash_including({
+            body: body,
+        })).and_return(
+          double(headers: {})
+        )
+        expect(subject.manticore).to receive(:post).
+          with(expected_uri.to_s, {
+            :body => encoded_body,
+            :headers => {"content-type"=> "application/json"}
+          }
         ).and_return resp
+        subject.perform_request(uri, :post, "/", { body: encoded_body })
+      end
 
-      expect(subject).to receive(:sign_aws_request)
-      subject.perform_request(uri, :get, "/")
+      it 'encodes body before signing to match manticore adapter encoding' do
+        body = "boîte de réception"
+        encoded_body = body.encode("ISO-8859-1")
+        expect_any_instance_of(Aws::Sigv4::Signer).to receive(:sign_request).with(hash_including({
+            body: body,
+        })).and_return(
+          double(headers: {})
+        )
+        expect(subject.manticore).to receive(:post).
+          with(expected_uri.to_s, {
+            :body => encoded_body,
+            :headers => {"content-type"=> "application/json"}
+          }
+        ).and_return resp
+        subject.perform_request(uri, :post, "/", { body: encoded_body })
+      end
     end
   end
 

data/spec/unit/outputs/opensearch/http_client_spec.rb

@@ -162,6 +162,26 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
     end
   end
 
+  describe "legacy_template" do
+    let(:template_name) { "logstash" }
+    let(:template) { {} }
+    let(:get_response) {
+      double("response", :body => {}, :code => 404)
+    }
+    [true, false].each do |legacy_template|
+      context "when legacy_template => #{legacy_template}" do
+        let(:base_options) { super().merge(:client_settings => {:legacy_template => legacy_template}) }
+        subject { described_class.new(base_options) }
+        endpoint = legacy_template ? "_template" : "_index_template"
+        it "should call template_endpoint #{endpoint}" do
+          expect(subject.pool).to receive(:head).with("/#{endpoint}/#{template_name}").and_return(get_response)
+          expect(subject.pool).to receive(:put).with("/#{endpoint}/#{template_name}", nil, anything).and_return(get_response)  
+          subject.template_install(template_name, template)
+        end
+      end
+    end
+  end
+
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

data/spec/unit/outputs/opensearch/template_manager_spec.rb

@@ -13,27 +13,15 @@ require "logstash/outputs/opensearch/template_manager"
 describe LogStash::Outputs::OpenSearch::TemplateManager do
 
   describe ".default_template_path" do
-    [1, 2].each do |major_version|
-      context "when ECS is disabled with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
-        end
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
-        end
-      end
-    end
     [7, 1, 2].each do |major_version|
-      context "when ECS v1 is requested with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :v1)).to end_with("/templates/ecs-v1/#{major_version}x.json")
-        end
-      end
-    end
-    [1, 2].each do |major_version|
-      context "when ECS v8 is requested with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
+      [:disabled, :v1, :v8].each do |ecs_ver|
+        [true, false].each do |legacy_template|
+          context "when ECS is #{ecs_ver} with OpenSearch #{major_version}.x legacy_template:#{legacy_template}" do
+            suffix = legacy_template ? "" : "_index"
+            it 'resolves' do
+              expect(described_class.default_template_path(major_version, ecs_ver, legacy_template)).to end_with("/templates/ecs-#{ecs_ver}/#{major_version}x#{suffix}.json")
+            end
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 2.0.2
 platform: java
 authors:
 - Elastic
@@ -11,9 +11,9 @@ bindir: bin
 cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
-  MIIDfDCCAmSgAwIBAgIBATANBgkqhkiG9w0BAQUFADBCMRMwEQYDVQQDDApvcGVu
+  MIIDfDCCAmSgAwIBAgIBATANBgkqhkiG9w0BAQsFADBCMRMwEQYDVQQDDApvcGVu
   c2VhcmNoMRYwFAYKCZImiZPyLGQBGRYGYW1hem9uMRMwEQYKCZImiZPyLGQBGRYD
-  Y29tMB4XDTIyMDgxNzE3NTIzNFoXDTIzMDgxNzE3NTIzNFowQjETMBEGA1UEAwwK
+  Y29tMB4XDTIzMDgyNDIwNDIwNFoXDTI0MDgyMzIwNDIwNFowQjETMBEGA1UEAwwK
   b3BlbnNlYXJjaDEWMBQGCgmSJomT8ixkARkWBmFtYXpvbjETMBEGCgmSJomT8ixk
   ARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1z3/jcitjV
   umXwRFn+JSBBd36qZB54Dtucf6+E2fmNPzBRhgYN5XJy/+clQJ9NIJV7C8H52P3V
@@ -24,14 +24,14 @@ cert_chain:
   zfR37/NQFkECAwEAAaN9MHswCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAwHQYDVR0O
   BBYEFJJ2myhLXK742btavNbG0IWrMNBIMCAGA1UdEQQZMBeBFW9wZW5zZWFyY2hA
   YW1hem9uLmNvbTAgBgNVHRIEGTAXgRVvcGVuc2VhcmNoQGFtYXpvbi5jb20wDQYJ
-  KoZIhvcNAQEFBQADggEBAH5pWLYwKWFh1OjdCReGz/VEAiF4jXXputoN5Z8ga+1Z
-  lg8/diJf0PlP2B46PdmxH/TVc/o+qglNO2cHVEp8xZfEd83dfioOBeK90URQUpC5
-  UZmO0LZusg46SQKwKa2ukpIy2fNi3PeHRiV+W2Zv69GoWppyLun+fMez7wVoah2r
-  r5ROUYuAvFUvga1Vm+49pKiPM5n+MAYP5t/vWhgymY3SYQ1TfewkvKAFiFXikOR+
-  r+j7FLyKuk5DzIxiCp8QN5dU71BbGUmsHf/C5UV76WLPOFX/szeaHhPwpjR3sK7r
-  5zLgCV1KP7cgDdCYMlmZGeSViU8NV+Yy8/ghrzGpqVw=
+  KoZIhvcNAQELBQADggEBABAQpnELuY5AgdNUIlIVRVATO6iZOXTbt3a9oVbQdLPe
+  BfMObZyJydg0+leyR3oFyN9ZIFiEFwpd0biFf39DuC0M6Oge0Rv4oO9GRI3yyv77
+  9m59he+5DI3hbqtGje108oqRe61NZMlKcy/DCBVkzzZFsJ17GC09tY/gwhmNRtLV
+  3vYIEY6vmn57wrGn1NUzWdG+x/XVjYPw5Kwo+/rCxxZqpVTklMqVWV43N/lFrUOe
+  1DlemA1SsUBIoF7CwtVd/RTG/K1iT6nBD08fdKxodMhI5ujkP3N7gkxzRf6aKN4z
+  glnDJYZjluKBUsKTOLdPW1CZpb0AHLpNqDf8SVHsPFk=
   -----END CERTIFICATE-----
-date: 2022-08-17 00:00:00.000000000 Z
+date: 2023-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -110,23 +110,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.11.632
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '3'
   name: aws-sdk
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.11.632
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -237,6 +231,7 @@ files:
 - README.md
 - RELEASING.md
 - SECURITY.md
+- docs/ecs_compatibility.md
 - lib/logstash/outputs/opensearch.rb
 - lib/logstash/outputs/opensearch/http_client.rb
 - lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb
@@ -244,11 +239,17 @@ files:
 - lib/logstash/outputs/opensearch/http_client_builder.rb
 - lib/logstash/outputs/opensearch/template_manager.rb
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json
 - lib/logstash/plugin_mixins/opensearch/api_configs.rb
 - lib/logstash/plugin_mixins/opensearch/common.rb
 - logstash-output-opensearch.gemspec
@@ -306,7 +307,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.20
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Stores logs in OpenSearch