logstash-output-opensearch 1.1.0-java → 2.0.0-java

data/spec/integration/outputs/templates_spec.rb

@@ -10,8 +10,12 @@
 require_relative "../../../spec/opensearch_spec_helper"
 
 describe "index template expected behavior", :integration => true do
+  let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
+
   subject! do
     require "logstash/outputs/opensearch"
+    allow_any_instance_of(LogStash::Outputs::OpenSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+
     settings = {
       "manage_template" => true,
       "template_overwrite" => true,
@@ -21,8 +25,6 @@ describe "index template expected behavior", :integration => true do
   end
 
   before :each do
-    # Delete all templates first.
-    require "elasticsearch"
 
     # Clean OpenSearch of data before we start.
     @client = get_client
@@ -31,75 +33,97 @@ describe "index template expected behavior", :integration => true do
     # This can fail if there are no indexes, ignore failure.
     @client.indices.delete(:index => "*") rescue nil
 
-    subject.register
-
-    subject.multi_receive([
-      LogStash::Event.new("message" => "sample message here"),
-      LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
-      LogStash::Event.new("somevalue" => 100),
-      LogStash::Event.new("somevalue" => 10),
-      LogStash::Event.new("somevalue" => 1),
-      LogStash::Event.new("country" => "us"),
-      LogStash::Event.new("country" => "at"),
-      LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
-    ])
-
-    @client.indices.refresh
-
-    # Wait or fail until everything's indexed.
-    Stud::try(20.times) do
-      r = @client.search(index: 'logstash-*')
-      expect(r).to have_hits(8)
-    end
   end
 
-  it "permits phrase searching on string fields" do
-    results = @client.search(:q => "message:\"sample message\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
-  end
+  context 'with ecs_compatibility => disabled' do
+    let(:ecs_compatibility) { :disabled }
+
+    before :each do
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @client.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @client.search(index: 'logstash-*')
+        expect(r).to have_hits(8)
+      end
+    end
 
-  it "numbers dynamically map to a numeric type and permit range queries" do
-    results = @client.search(:q => "somevalue:[5 TO 105]")
-    expect(results).to have_hits(2)
+    it "permits phrase searching on string fields" do
+      results = @client.search(:q => "message:\"sample message\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
+    end
 
-    values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
-    expect(values).to include(10)
-    expect(values).to include(100)
-    expect(values).to_not include(1)
-  end
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @client.search(:q => "somevalue:[5 TO 105]")
+      expect(results).to have_hits(2)
 
-  it "does not create .keyword field for top-level message field" do
-    results = @client.search(:q => "message.keyword:\"sample message here\"")
-    expect(results).to have_hits(0)
-  end
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      expect(values).to include(10)
+      expect(values).to include(100)
+      expect(values).to_not include(1)
+    end
 
-  it "creates .keyword field for nested message fields" do
-    results = @client.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
-    expect(results).to have_hits(1)
-  end
+    it "does not create .keyword field for top-level message field" do
+      results = @client.search(:q => "message.keyword:\"sample message here\"")
+      expect(results).to have_hits(0)
+    end
 
-  it "creates .keyword field from any string field which is not_analyzed" do
-    results = @client.search(:q => "country.keyword:\"us\"")
-    expect(results).to have_hits(1)
-    expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+    it "creates .keyword field for nested message fields" do
+      results = @client.search(:q => "somemessage.message.keyword:\"sample nested message here\"")
+      expect(results).to have_hits(1)
+    end
 
-    # partial or terms should not work.
-    results = @client.search(:q => "country.keyword:\"u\"")
-    expect(results).to have_hits(0)
-  end
+    it "creates .keyword field from any string field which is not_analyzed" do
+      results = @client.search(:q => "country.keyword:\"us\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+
+      # partial or terms should not work.
+      results = @client.search(:q => "country.keyword:\"u\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "make [geoip][location] a geo_point" do
+      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    end
 
-  it "make [geoip][location] a geo_point" do
-    expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    it "aggregate .keyword results correctly " do
+      results = @client.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
+
+      expect(terms).to include("us")
+
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      expect(terms).to include("at")
+    end
   end
 
-  it "aggregate .keyword results correctly " do
-    results = @client.search(:body => { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
-    terms = results["buckets"].collect { |b| b["key"] }
+  context 'with ECS enabled' do
+    let(:ecs_compatibility) { :v1 }
 
-    expect(terms).to include("us")
+    before(:each) do
+      subject.register # should load template?
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+    end
 
-    # 'at' is a stopword, make sure stopwords are not ignored.
-    expect(terms).to include("at")
+    it 'loads the templates' do
+      aggregate_failures do
+        expect(@client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
+      end
+    end
   end
 end

data/spec/integration/outputs/update_spec.rb

@@ -12,7 +12,7 @@ require_relative "../../../spec/opensearch_spec_helper"
 describe "Update actions without scripts", :integration => true do
   require "logstash/outputs/opensearch"
 
-  def get_es_output( options={} )
+  def get_output( options={} )
     settings = {
       "manage_template" => true,
       "index" => "logstash-update",
@@ -40,20 +40,20 @@ describe "Update actions without scripts", :integration => true do
   end
 
   it "should fail without a document_id" do
-    subject = get_es_output
+    subject = get_output
     expect { subject.register }.to raise_error(LogStash::ConfigurationError)
   end
 
   context "when update only" do
     it "should not create new document" do
-      subject = get_es_output({ 'document_id' => "456" } )
+      subject = get_output({ 'document_id' => "456" } )
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-      expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      expect {@client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(OpenSearch::Transport::Transport::Errors::NotFound)
     end
 
     it "should update existing document" do
-      subject = get_es_output({ 'document_id' => "123" })
+      subject = get_output({ 'document_id' => "123" })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -63,7 +63,7 @@ describe "Update actions without scripts", :integration => true do
     # The es ruby client treats the data field differently. Make sure this doesn't
     # raise an exception
     it "should update an existing document that has a 'data' field" do
-      subject = get_es_output({ 'document_id' => "123" })
+      subject = get_output({ 'document_id' => "123" })
       subject.register
       subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
@@ -72,27 +72,27 @@ describe "Update actions without scripts", :integration => true do
     end
 
     it "should allow default (internal) version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+      subject = get_output({ 'document_id' => "123", "version" => "99" })
       subject.register
     end
 
     it "should allow internal version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
       subject.register
     end
 
     it "should not allow external version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gt version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
     it "should not allow external_gte version" do
-      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+      subject = get_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
 
@@ -100,7 +100,7 @@ describe "Update actions without scripts", :integration => true do
 
   context "when update with upsert" do
     it "should create new documents with provided upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject = get_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -108,7 +108,7 @@ describe "Update actions without scripts", :integration => true do
     end
 
     it "should create new documents with event/doc as upsert" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
       r = @client.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
@@ -116,7 +116,7 @@ describe "Update actions without scripts", :integration => true do
     end
 
     it "should fail on documents with event/doc as upsert at external version" do
-      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      subject = get_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
   end

data/spec/opensearch_spec_helper.rb

@@ -9,7 +9,7 @@
 
 require_relative './spec_helper'
 
-require 'elasticsearch'
+require 'opensearch'
 
 require 'json'
 require 'cabin'
@@ -24,7 +24,7 @@ module OpenSearchHelper
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => [get_host_port])
+    OpenSearch::Client.new(:hosts => [get_host_port])
   end
 
   def doc_type
@@ -49,6 +49,16 @@ module OpenSearchHelper
     RSpec.configuration.filter[:version]
   end
 
+  def self.check_version?(*requirement)
+    version = self.version
+    if version.nil? || version.empty?
+      puts "version tag isn't set. Returning false from 'check_version?`."
+      return false
+    end
+    release_version = Gem::Version.new(version).release
+    Gem::Requirement.new(requirement).satisfied_by?(release_version)
+  end
+
   RSpec::Matchers.define :have_hits do |expected|
     match do |actual|
       expected == actual['hits']['total']['value']

data/spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb

@@ -25,13 +25,13 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
   it "should implement host unreachable exceptions" do
     expect(subject.host_unreachable_exceptions).to be_a(Array)
   end
-  
+
   describe "auth" do
     let(:user) { "myuser" }
     let(:password) { "mypassword" }
     let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
     let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
-    
+
     it "should convert the auth to params" do
       resp = double("response")
       allow(resp).to receive(:call)
@@ -39,7 +39,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
       
       expected_uri = noauth_uri.clone
       expected_uri.path = "/"
-      
+
       expect(subject.manticore).to receive(:get).
         with(expected_uri.to_s, {
           :headers => {"content-type" => "application/json"},
@@ -54,6 +54,89 @@ describe LogStash::Outputs::OpenSearch::HttpClient::ManticoreAdapter do
     end
   end
 
+  describe "aws_iam" do
+    let(:options) { {
+      :auth_type => {
+        "type"=>"aws_iam",
+        "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
+        "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}
+    } }
+    let(:options_svc) { {
+      :auth_type => {
+        "type"=>"aws_iam",
+        "aws_access_key_id"=>"AAAAAAAAAAAAAAAAAAAA",
+        "aws_secret_access_key"=>"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+        "service_name"=>"svc_test"}
+    } }
+    subject { described_class.new(logger, options) }
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+    let(:sign_aws_request) {  }
+
+    it "should validate AWS IAM credentials initialization" do
+      expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
+      expect(subject.get_service_name).to eq("es") 
+    end
+
+    it "should validate AWS IAM service_name config" do
+      expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
+      expect(subject.get_service_name).to eq("svc_test") 
+    end
+
+    it "should validate signing aws request" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(200)
+      allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+
+      expected_uri = uri.clone
+      expected_uri.path = "/"
+
+      expect(subject.manticore).to receive(:get).
+        with(expected_uri.to_s, {
+          :headers => {"content-type"=> "application/json"}
+        }
+        ).and_return resp
+
+      expect(subject).to receive(:sign_aws_request)
+      subject.perform_request(uri, :get, "/")
+    end
+  end
+
+  describe "basic_auth" do
+    let(:options) { {
+      :auth_type => {
+        "type"=>"basic",
+        "user" => "myuser",
+        "password" => "mypassword"}
+    } }
+    subject { described_class.new(logger, options) }
+    let(:user) {options[:auth_type]["user"]}
+    let(:password) {options[:auth_type]["password"]}
+    let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+
+    it "should validate master credentials with type as 'basic_auth'" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(200)
+
+      expected_uri = noauth_uri.clone
+      expected_uri.path = "/"
+
+      expect(subject.manticore).to receive(:get).
+        with(expected_uri.to_s, {
+          :headers => {"content-type" => "application/json"},
+          :auth => {
+            :user => user,
+            :password => password,
+            :eager => true
+          }
+        }).and_return resp
+
+      subject.perform_request(uri, :get, "/")
+    end
+  end
+
   describe "bad response codes" do
     let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
 

data/spec/unit/outputs/opensearch/http_client_spec.rb

@@ -17,6 +17,7 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
     opts = {
       :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
       :logger => Cabin::Channel.get,
+      :target_bulk_bytes => 9_000_000,
       :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
     }
 
@@ -156,11 +157,31 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
     }
 
     it "should call index template" do
-      expect(subject.pool).to receive(:put).with("_template/#{template_name}", nil, anything).and_return(get_response)
+      expect(subject.pool).to receive(:put).with("/_template/#{template_name}", nil, anything).and_return(get_response)
       subject.template_put(template_name, template)
     end
   end
 
+  describe "legacy_template" do
+    let(:template_name) { "logstash" }
+    let(:template) { {} }
+    let(:get_response) {
+      double("response", :body => {}, :code => 404)
+    }
+    [true, false].each do |legacy_template|
+      context "when legacy_template => #{legacy_template}" do
+        let(:base_options) { super().merge(:client_settings => {:legacy_template => legacy_template}) }
+        subject { described_class.new(base_options) }
+        endpoint = legacy_template ? "_template" : "_index_template"
+        it "should call template_endpoint #{endpoint}" do
+          expect(subject.pool).to receive(:head).with("/#{endpoint}/#{template_name}").and_return(get_response)
+          expect(subject.pool).to receive(:put).with("/#{endpoint}/#{template_name}", nil, anything).and_return(get_response)  
+          subject.template_install(template_name, template)
+        end
+      end
+    end
+  end
+
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 
@@ -226,8 +247,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
           end
         end
 
-        context "if a message is over TARGET_BULK_BYTES" do
-          let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+        context "if a message is over target_bulk_bytes" do
+          let(:target_bulk_bytes) { subject.target_bulk_bytes }
           let(:message) { "a" * (target_bulk_bytes + 1) }
 
           it "should be handled properly" do
@@ -256,8 +277,8 @@ describe LogStash::Outputs::OpenSearch::HttpClient do
             s = subject.send(:bulk, actions)
           end
 
-          context "if one exceeds TARGET_BULK_BYTES" do
-            let(:target_bulk_bytes) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES }
+          context "if one exceeds target_bulk_bytes" do
+            let(:target_bulk_bytes) { subject.target_bulk_bytes }
             let(:message1) { "a" * (target_bulk_bytes + 1) }
             it "executes two bulk_send operations" do
               allow(subject).to receive(:join_bulk_responses)

data/spec/unit/outputs/opensearch/template_manager_spec.rb

@@ -13,9 +13,16 @@ require "logstash/outputs/opensearch/template_manager"
 describe LogStash::Outputs::OpenSearch::TemplateManager do
 
   describe ".default_template_path" do
-    context 'when ECS v1 is requested' do
-      it 'resolves' do
-        expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/7x.json")
+    [7, 1, 2].each do |major_version|
+      [:disabled, :v1, :v8].each do |ecs_ver|
+        [true, false].each do |legacy_template|
+          context "when ECS is #{ecs_ver} with OpenSearch #{major_version}.x legacy_template:#{legacy_template}" do
+            suffix = legacy_template ? "" : "_index"
+            it 'resolves' do
+              expect(described_class.default_template_path(major_version, ecs_ver, legacy_template)).to end_with("/templates/ecs-#{ecs_ver}/#{major_version}x#{suffix}.json")
+            end
+          end
+        end
       end
     end
   end
@@ -33,4 +40,4 @@ describe LogStash::Outputs::OpenSearch::TemplateManager do
       end
     end
   end
-end
+end

data/spec/unit/outputs/opensearch_spec.rb

@@ -12,6 +12,7 @@ require "base64"
 require "flores/random"
 require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/opensearch"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
 
 describe LogStash::Outputs::OpenSearch do
   subject(:opensearch_output_instance) { described_class.new(options) }
@@ -325,7 +326,7 @@ describe LogStash::Outputs::OpenSearch do
   end
 
   context '413 errors' do
-    let(:payload_size) { LogStash::Outputs::OpenSearch::TARGET_BULK_BYTES + 1024 }
+    let(:payload_size) { subject.client.target_bulk_bytes + 1024 }
     let(:event) { ::LogStash::Event.new("message" => ("a" * payload_size ) ) }
 
     let(:logger_stub) { double("logger").as_null_object }
@@ -357,7 +358,7 @@ describe LogStash::Outputs::OpenSearch do
 
       expect(logger_stub).to have_received(:warn)
                                  .with(a_string_matching(/413 Payload Too Large/),
-                                       hash_including(:action_count => 1, :content_length => a_value > 20_000_000))
+                                       hash_including(:action_count => 1, :content_length => a_value > subject.client.target_bulk_bytes))
     end
   end
 
@@ -784,6 +785,21 @@ describe LogStash::Outputs::OpenSearch do
     end
   end
 
+  describe 'ecs_compatibility support', :ecs_compatibility_support do
+    [:disabled, :v1, :v8].each do |ecs_compatibility|
+      context "when `ecs_compatibility => #{ecs_compatibility}`" do
+        let(:options) { Hash.new }
+        subject(:output) { described_class.new(options.merge("ecs_compatibility" => "#{ecs_compatibility}")) }
+        context 'when registered' do
+          before(:each) { output.register }
+          it 'has the correct effective ECS compatibility setting' do
+            expect(output.ecs_compatibility).to eq(ecs_compatibility)
+          end
+        end
+      end
+    end
+  end
+  
   @private
 
   def stub_manticore_client!(manticore_double = nil)