logstash-output-elasticsearch-test 11.16.0-x86_64-linux

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -0,0 +1,339 @@
+require_relative "../../../../spec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+require "cabin"
+require "webrick"
+require "java"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient do
+  let(:ssl) { nil }
+  let(:base_options) do
+    opts = {
+      :hosts => [::LogStash::Util::SafeURI.new("127.0.0.1")],
+      :logger => Cabin::Channel.get,
+      :metric => ::LogStash::Instrument::NullMetric.new(:dummy).namespace(:alsodummy)
+    }
+
+    if !ssl.nil? # Shortcut to set this
+      opts[:client_settings] = {:ssl => {:enabled => ssl}}
+    end
+
+    opts
+  end
+
+  describe "Host/URL Parsing" do
+    subject { described_class.new(base_options) }
+
+    let(:true_hostname) { "my-dash.hostname" }
+    let(:ipv6_hostname) { "[::1]" }
+    let(:ipv4_hostname) { "127.0.0.1" }
+    let(:port) { 9202 }
+    let(:hostname_port) { "#{hostname}:#{port}" }
+    let(:hostname_port_uri) { ::LogStash::Util::SafeURI.new("//#{hostname_port}") }
+    let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") }
+    let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") }
+    let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") }
+    
+    shared_examples("proper host handling") do
+      it "should properly transform a host:port string to a URL" do
+        expect(subject.host_to_url(hostname_port_uri).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+      it "should not raise an error with a / for a path" do
+        expect(subject.host_to_url(::LogStash::Util::SafeURI.new("#{http_hostname_port}/"))).to eq(LogStash::Util::SafeURI.new("#{http_hostname_port}/"))
+      end
+
+      it "should parse full URLs correctly" do
+        expect(subject.host_to_url(http_hostname_port).to_s).to eq(http_hostname_port.to_s + "/")
+      end
+
+      describe "ssl" do
+        context "when SSL is true" do
+          let(:ssl) { true }
+          let(:base_options) { super().merge(:hosts => [http_hostname_port]) }
+
+          it "should refuse to handle an http url" do
+            expect {
+              subject.host_to_url(http_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+
+        context "when SSL is false" do
+          let(:ssl) { false }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
+          
+          it "should refuse to handle an https url" do
+            expect {
+              subject.host_to_url(https_hostname_port)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+
+        describe "ssl is nil" do
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
+          it "should handle an ssl url correctly when SSL is nil" do
+            subject
+            expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
+          end
+        end       
+      end
+
+      describe "path" do
+        let(:url) { http_hostname_port_path }
+        let(:base_options) { super().merge(:hosts => [url]) }
+        
+        it "should allow paths in a url" do
+          expect(subject.host_to_url(url)).to eq(url)
+        end
+
+        context "with the path option set" do
+          let(:base_options) { super().merge(:client_settings => {:path => "/otherpath"}) }
+
+          it "should not allow paths in two places" do
+            expect {
+              subject.host_to_url(url)
+            }.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+        
+        context "with a path missing a leading /" do
+          let(:url) { http_hostname_port }
+          let(:base_options) { super().merge(:client_settings => {:path => "otherpath"}) }
+          
+          
+          it "should automatically insert a / in front of path overlays" do
+            expected = url.clone
+            expected.path = url.path + "/otherpath"
+            expect(subject.host_to_url(url)).to eq(expected)
+          end
+        end
+      end
+    end
+
+    describe "an regular hostname" do
+      let(:hostname) { true_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv4 host" do
+      let(:hostname) { ipv4_hostname }
+      include_examples("proper host handling")
+    end
+
+    describe "an ipv6 host" do
+      let(:hostname) { ipv6_hostname }
+      include_examples("proper host handling")
+    end
+  end
+
+  describe "get" do
+    subject { described_class.new(base_options) }
+    let(:body) { "foobar" }
+    let(:path) { "/hello-id" }
+    let(:get_response) {
+      double("response", :body => LogStash::Json::dump( { "body" => body }))
+    }
+
+    it "returns the hash response" do
+      expect(subject.pool).to receive(:get).with(path, nil).and_return(get_response)
+      expect(subject.get(path)["body"]).to eq(body)
+    end
+  end
+
+  describe "join_bulk_responses" do
+    subject { described_class.new(base_options) }
+
+    context "when items key is available" do
+      require "json"
+      let(:bulk_response) {
+        LogStash::Json.load ('[{
+          "items": [{
+            "delete": {
+              "_index":   "website",
+              "_type":    "blog",
+              "_id":      "123",
+              "_version": 2,
+              "status":   200,
+              "found":    true
+            }
+          }],
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 1
+      end
+    end
+
+    context "when items key is not available" do
+      require "json"
+      let(:bulk_response) {
+        JSON.parse ('[{
+          "took": 4,
+          "errors": false
+        }]')
+      }
+      it "should be handled properly" do
+        s = subject.send(:join_bulk_responses, bulk_response)
+        expect(s["errors"]).to be false
+        expect(s["items"].size).to be 0
+      end
+    end
+  end
+
+  describe "#bulk" do
+    subject(:http_client) { described_class.new(base_options) }
+
+    require "json"
+    let(:message) { "hey" }
+    let(:actions) { [
+      ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
+    ]}
+
+    [true,false].each do |http_compression_enabled|
+      context "with `http_compression => #{http_compression_enabled}`" do
+
+        let(:base_options) { super().merge(:client_settings => {:http_compression => http_compression_enabled}) }
+
+        before(:each) do
+          if http_compression_enabled
+            expect(http_client).to receive(:gzip_writer).at_least(:once).and_call_original
+          else
+            expect(http_client).to_not receive(:gzip_writer)
+          end
+        end
+
+        context "if a message is over TARGET_BULK_BYTES" do
+          let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+          let(:message) { "a" * (target_bulk_bytes + 1) }
+
+          it "should be handled properly" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once do |data|
+              if !http_compression_enabled
+                expect(data.size).to be > target_bulk_bytes
+              else
+                expect(Zlib::gunzip(data.string).size).to be > target_bulk_bytes
+              end
+            end
+            s = subject.send(:bulk, actions)
+          end
+        end
+
+        context "with two messages" do
+          let(:message1) { "hey" }
+          let(:message2) { "you" }
+          let(:actions) { [
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+          ]}
+          it "executes one bulk_send operation" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once
+            s = subject.send(:bulk, actions)
+          end
+
+          context "if one exceeds TARGET_BULK_BYTES" do
+            let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+            let(:message1) { "a" * (target_bulk_bytes + 1) }
+            it "executes two bulk_send operations" do
+              allow(subject).to receive(:join_bulk_responses)
+              expect(subject).to receive(:bulk_send).twice
+              s = subject.send(:bulk, actions)
+            end
+          end
+        end
+
+       end
+     end
+  end
+
+  describe "sniffing" do
+    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
+
+    context "with sniffing enabled" do
+      let(:client_opts) { {:sniffing => true, :sniffing_delay => 1 } }
+
+      it "should start the sniffer" do
+        expect(client.pool.sniffing).to be_truthy
+      end
+    end
+
+    context "with sniffing disabled" do
+      let(:client_opts) { {:sniffing => false} }
+
+      it "should not start the sniffer" do
+        expect(client.pool.sniffing).to be_falsey
+      end
+    end
+  end
+
+  class StoppableServer
+
+    attr_reader :port
+
+    def initialize()
+      queue = Queue.new
+      @first_req_waiter = java.util.concurrent.CountDownLatch.new(1)
+      @first_request = nil
+
+      @t = java.lang.Thread.new(
+        proc do
+          begin
+            @server = WEBrick::HTTPServer.new :Port => 0, :DocumentRoot => ".",
+                     :Logger => Cabin::Channel.get, # silence WEBrick logging
+                     :StartCallback => Proc.new {
+                           queue.push("started")
+                         }
+            @port = @server.config[:Port]
+            @server.mount_proc '/headers_check' do |req, res|
+              res.body = 'Hello, world from WEBrick mocking server!'
+              @first_request = req
+              @first_req_waiter.countDown()
+            end
+
+            @server.start
+          rescue => e
+            puts "Error in webserver thread #{e}"
+            # ignore
+          end
+        end
+      )
+      @t.daemon = true
+      @t.start
+      queue.pop # blocks until the server is up
+    end
+
+    def stop
+      @server.shutdown
+    end
+
+    def wait_receive_request
+      @first_req_waiter.await(2, java.util.concurrent.TimeUnit::SECONDS)
+      @first_request
+    end
+  end
+
+  describe "#build_adapter" do
+    let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options) }
+    let!(:webserver) { StoppableServer.new } # webserver must be started before the call, so no lazy "let"
+
+    after :each do
+      webserver.stop
+    end
+
+    context "the 'user-agent' header" do
+      it "contains the Logstash environment details" do
+        adapter = client.build_adapter(client.options)
+        adapter.perform_request(::LogStash::Util::SafeURI.new("http://localhost:#{webserver.port}"), :get, "/headers_check")
+
+        request = webserver.wait_receive_request
+
+        transmitted_user_agent = request.header['user-agent'][0]
+        expect(transmitted_user_agent).to match(/Logstash\/\d*\.\d*\.\d* \(OS=.*; JVM=.*\) logstash-output-elasticsearch\/\d*\.\d*\.\d*/)
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/template_manager_spec.rb

@@ -0,0 +1,189 @@
+require_relative "../../../../spec/spec_helper"
+require "logstash/outputs/elasticsearch/template_manager"
+
+describe LogStash::Outputs::ElasticSearch::TemplateManager do
+
+  describe ".default_template_path" do
+    context "elasticsearch 6.x" do
+      it "chooses the 6x template" do
+        expect(described_class.default_template_path(6)).to end_with("/templates/ecs-disabled/elasticsearch-6x.json")
+      end
+    end
+    context "elasticsearch 7.x" do
+      it "chooses the 7x template" do
+        expect(described_class.default_template_path(7)).to end_with("/templates/ecs-disabled/elasticsearch-7x.json")
+      end
+    end
+    context "elasticsearch 8.x" do
+      it "chooses the 8x template" do
+        expect(described_class.default_template_path(8)).to end_with("/templates/ecs-disabled/elasticsearch-8x.json")
+      end
+    end
+  end
+
+  context 'when ECS v1 is requested' do
+    it 'resolves' do
+      expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/elasticsearch-7x.json")
+    end
+  end
+
+  context 'when ECS v8 is requested' do
+    it 'resolves' do
+      expect(described_class.default_template_path(7, :v8)).to end_with("/templates/ecs-v8/elasticsearch-7x.json")
+    end
+  end
+
+  context "index template with ilm settings" do
+    let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} }
+    let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+    describe "with custom template" do
+
+      describe "in version 8+" do
+        let(:file_path) { described_class.default_template_path(8) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('settings')).to be_falsey
+        end
+      end
+
+      describe "in version < 8" do
+        let(:file_path) { described_class.default_template_path(7) }
+        let(:template) { described_class.read_template_file(file_path)}
+
+        it "should update settings" do
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          described_class.add_ilm_settings_to_template(plugin, template)
+          expect(template['settings']['index.lifecycle.name']).not_to eq(nil)
+          expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil)
+          expect(template.include?('template')).to be_falsey
+        end
+      end
+    end
+
+    context "resolve template setting" do
+      let(:plugin_settings) { super().merge({"template_api" => template_api}) }
+
+      describe "with composable template API" do
+        let(:template_api) { "composable" }
+
+        it 'resolves composable index template API compatible setting' do
+          expect(plugin).to receive(:serverless?).and_return(false)
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["template"]["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with legacy template API" do
+        let(:template_api) { "legacy" }
+
+        it 'resolves legacy index template API compatible setting' do
+          expect(plugin).to receive(:serverless?).and_return(false)
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log
+          template = {}
+          described_class.resolve_template_settings(plugin, template)
+          expect(template["settings"]).not_to eq(nil)
+        end
+      end
+
+      describe "with `template_api => 'auto'`" do
+        let(:template_api) { "auto" }
+
+        describe "with ES < 8 versions" do
+
+          it 'resolves legacy index template API compatible setting' do
+            expect(plugin).to receive(:serverless?).and_return(false)
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["settings"]).not_to eq(nil)
+          end
+        end
+
+        describe "with ES >= 8 versions" do
+          it 'resolves composable index template API compatible setting' do
+            expect(plugin).to receive(:serverless?).and_return(false)
+            expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+            template = {}
+            described_class.resolve_template_settings(plugin, template)
+            expect(template["template"]["settings"]).not_to eq(nil)
+          end
+        end
+      end
+    end
+  end
+
+  describe "template endpoint" do
+    describe "template_api => 'auto'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'auto'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use index template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+
+      describe "in version < 8" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
+          expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+
+    describe "template_api => 'legacy'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'legacy'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
+          expect(plugin).to receive(:maximum_seen_major_version).never
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::LEGACY_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+
+    describe "template_api => 'composable'" do
+      let(:plugin_settings) { {"manage_template" => true, "template_api" => 'composable'} }
+      let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+      describe "in version 8+" do
+        it "should use legacy template API" do
+          expect(plugin).to receive(:serverless?).and_return(false)
+          expect(plugin).to receive(:maximum_seen_major_version).never
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+    end
+
+    describe "in serverless" do
+      [:auto, :composable, :legacy].each do |api|
+        let(:plugin_settings) { {"manage_template" => true, "template_api" => api.to_s} }
+        let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) }
+
+        it "use index template API when template_api set to #{api}" do
+          expect(plugin).to receive(:serverless?).and_return(true)
+          endpoint = described_class.template_endpoint(plugin)
+          expect(endpoint).to be_equal(LogStash::Outputs::ElasticSearch::TemplateManager::INDEX_TEMPLATE_ENDPOINT)
+        end
+      end
+
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch_proxy_spec.rb

@@ -0,0 +1,103 @@
+require_relative "../../../spec/spec_helper"
+require 'stud/temporary'
+require 'manticore/client'
+
+describe "Proxy option" do
+  let(:settings) { { "hosts" => "node01" } }
+  subject {
+    LogStash::Outputs::ElasticSearch.new(settings)
+  }
+
+  before do
+    allow(::Manticore::Client).to receive(:new).with(any_args).and_call_original
+  end
+
+  describe "valid configs" do
+    before do
+      subject.register
+    end
+
+    after do
+      subject.close
+    end
+
+    context "when specified as a URI" do
+      shared_examples("hash conversion") do |hash|
+        let(:settings) { super().merge("proxy" => proxy)}
+        
+        it "should set the proxy to the correct hash value" do
+          expect(::Manticore::Client).to have_received(:new) do |options|
+            expect(options[:proxy]).to eq(hash)
+          end
+        end
+      end
+      
+      describe "simple proxy" do
+        let(:proxy) { LogStash::Util::SafeURI.new("http://127.0.0.1:1234") }
+
+        include_examples("hash conversion",
+          {
+            :host => "127.0.0.1",
+            :scheme => "http",
+            :port => 1234  
+          }
+        )
+      end
+      
+      
+      describe "a secure authed proxy" do
+        let(:proxy) { LogStash::Util::SafeURI.new("https://myuser:mypass@127.0.0.1:1234") }
+
+        include_examples("hash conversion",
+          {
+            :host => "127.0.0.1",
+            :scheme => "https",
+            :user => "myuser",
+            :password => "mypass",
+            :port => 1234  
+          }
+        )
+      end
+    end
+
+    context "when not specified" do
+      it "should not send the proxy option to manticore" do
+        expect(::Manticore::Client).to have_received(:new) do |options|
+          expect(options).not_to include(:proxy)
+        end
+      end
+    end
+  end
+
+  context "when specified as ''" do
+    let(:settings) { super().merge("proxy" => "${A_MISSING_ENV_VARIABLE:}")}
+
+    it "should not send the proxy option to manticore" do
+      expect { subject.register }.not_to raise_error
+
+      expect(::Manticore::Client).to have_received(:new) do |options|
+        expect(options).not_to include(:proxy)
+      end
+
+      subject.close
+    end
+  end
+
+  context "when specified as invalid uri" do
+    let(:settings) { super().merge("proxy" => ":")}
+
+    it "should fail" do
+      # SafeURI isn't doing the proper exception wrapping for us, we can not simply :
+      # expect { subject.register }.to raise_error(ArgumentError, /URI is not valid/i)
+      begin
+        subject.register
+      rescue ArgumentError => e
+        expect(e.message).to match /URI is not valid/i
+      rescue java.net.URISyntaxException => e
+        expect(e.message).to match /scheme name/i
+      else
+        fail 'exception not raised'
+      end
+    end
+  end
+end