logstash-output-opensearch 1.3.0-java → 2.0.0-java

data/lib/logstash/outputs/opensearch.rb

@@ -199,6 +199,13 @@ class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   # Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
   # here like `pipeline => "%{INGEST_PIPELINE}"`
   config :pipeline, :validate => :string, :default => nil
+  
+  # When set to true, use legacy templates via the _template API
+  # When false, use index templates using the _index_template API
+  config :legacy_template, :validate => :boolean, :default => true
+
+  # The OpenSearch server major version to use when it's not available from the Healthcheck endpoint.
+  config :default_server_major_version, :validate => :number
 
   attr_reader :client
   attr_reader :default_index

data/logstash-output-opensearch.gemspec

@@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem"
 
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-opensearch'
-  s.version         = '1.3.0'
+  s.version         = '2.0.0'
 
   s.licenses        = ['Apache-2.0']
   s.summary         = "Stores logs in OpenSearch"
@@ -45,7 +45,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0'
-  s.add_runtime_dependency 'aws-sdk',  '>= 2.11.632', '~> 2'
+  s.add_runtime_dependency 'aws-sdk', '~> 3'
   s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2'
 
   s.add_development_dependency 'logstash-codec-plain'

data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb

@@ -61,12 +61,25 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
         "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
         "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
     } }
+    let(:options_svc) { {
+      :auth_type => {
+        "type"=>"aws_iam",
+        "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
+        "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        "service_name"=>"svc_test"}
+    } }
     subject { described_class.new(logger, options) }
     let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
     let(:sign_aws_request) {  }
 
     it "should validate AWS IAM credentials initialization" do
       expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
+      expect(subject.get_service_name).to eq("es") 
+    end
+
+    it "should validate AWS IAM service_name config" do
+      expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
+      expect(subject.get_service_name).to eq("svc_test") 
     end
 
     it "should validate signing aws request" do

data/spec/unit/outputs/opensearch/http_client_spec.rb

@@ -162,6 +162,26 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
     end
   end
 
+  describe "legacy_template" do
+    let(:template_name) { "logstash" }
+    let(:template) { {} }
+    let(:get_response) {
+      double("response", :body => {}, :code => 404)
+    }
+    [true, false].each do |legacy_template|
+      context "when legacy_template => #{legacy_template}" do
+        let(:base_options) { super().merge(:client_settings => {:legacy_template => legacy_template}) }
+        subject { described_class.new(base_options) }
+        endpoint = legacy_template ? "_template" : "_index_template"
+        it "should call template_endpoint #{endpoint}" do
+          expect(subject.pool).to receive(:head).with("/#{endpoint}/#{template_name}").and_return(get_response)
+          expect(subject.pool).to receive(:put).with("/#{endpoint}/#{template_name}", nil, anything).and_return(get_response)  
+          subject.template_install(template_name, template)
+        end
+      end
+    end
+  end
+
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

data/spec/unit/outputs/opensearch/template_manager_spec.rb

@@ -13,27 +13,15 @@ require "logstash/outputs/opensearch/template_manager"
 describe LogStash::Outputs::OpenSearch::TemplateManager do
 
   describe ".default_template_path" do
-    [1, 2].each do |major_version|
-      context "when ECS is disabled with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
-        end
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :disabled)).to end_with("/templates/ecs-disabled/#{major_version}x.json")
-        end
-      end
-    end
     [7, 1, 2].each do |major_version|
-      context "when ECS v1 is requested with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :v1)).to end_with("/templates/ecs-v1/#{major_version}x.json")
-        end
-      end
-    end
-    [1, 2].each do |major_version|
-      context "when ECS v8 is requested with OpenSearch #{major_version}.x" do
-        it 'resolves' do
-          expect(described_class.default_template_path(major_version, :v8)).to end_with("/templates/ecs-v8/#{major_version}x.json")
+      [:disabled, :v1, :v8].each do |ecs_ver|
+        [true, false].each do |legacy_template|
+          context "when ECS is #{ecs_ver} with OpenSearch #{major_version}.x legacy_template:#{legacy_template}" do
+            suffix = legacy_template ? "" : "_index"
+            it 'resolves' do
+              expect(described_class.default_template_path(major_version, ecs_ver, legacy_template)).to end_with("/templates/ecs-#{ecs_ver}/#{major_version}x#{suffix}.json")
+            end
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-opensearch
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 2.0.0
 platform: java
 authors:
 - Elastic
@@ -31,7 +31,7 @@ cert_chain:
   r+j7FLyKuk5DzIxiCp8QN5dU71BbGUmsHf/C5UV76WLPOFX/szeaHhPwpjR3sK7r
   5zLgCV1KP7cgDdCYMlmZGeSViU8NV+Yy8/ghrzGpqVw=
   -----END CERTIFICATE-----
-date: 2022-08-17 00:00:00.000000000 Z
+date: 2022-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -56,23 +56,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.17
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.17
   name: stud
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.17
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.17
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -110,43 +110,37 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.11.632
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '3'
   name: aws-sdk
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.11.632
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.3.0
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.0
   name: json
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 2.3.0
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -237,6 +231,7 @@ files:
 - README.md
 - RELEASING.md
 - SECURITY.md
+- docs/ecs_compatibility.md
 - lib/logstash/outputs/opensearch.rb
 - lib/logstash/outputs/opensearch/http_client.rb
 - lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb
@@ -244,11 +239,23 @@ files:
 - lib/logstash/outputs/opensearch/http_client_builder.rb
 - lib/logstash/outputs/opensearch/template_manager.rb
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/1x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/2x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-disabled/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-disabled/7x_index.json
+- lib/logstash/outputs/opensearch/templates/ecs-v1/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v1/1x_index.json
+- lib/logstash/outputs/opensearch/templates/ecs-v1/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v1/2x_index.json
+- lib/logstash/outputs/opensearch/templates/ecs-v1/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v1/7x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/1x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/1x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/2x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/2x_index.json
 - lib/logstash/outputs/opensearch/templates/ecs-v8/7x.json
+- lib/logstash/outputs/opensearch/templates/ecs-v8/7x_index.json
 - lib/logstash/plugin_mixins/opensearch/api_configs.rb
 - lib/logstash/plugin_mixins/opensearch/common.rb
 - logstash-output-opensearch.gemspec
@@ -306,7 +313,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.20
+rubyforge_project:
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: Stores logs in OpenSearch