logstash-output-elasticsearch-test 10.3.0-x86_64-linux

data/spec/integration/outputs/update_spec.rb

@@ -0,0 +1,116 @@
+require_relative "../../../spec/es_spec_helper"
+
+if ESHelper.es_version_satisfies?(">= 2")
+  describe "Update actions without scripts", :integration => true do
+    require "logstash/outputs/elasticsearch"
+
+    def get_es_output( options={} )
+      settings = {
+        "manage_template" => true,
+        "index" => "logstash-update",
+        "template_overwrite" => true,
+        "hosts" => get_host_port(),
+        "action" => "update"
+      }
+      LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+    end
+
+    before :each do
+      @es = get_client
+      # Delete all templates first.
+      # Clean ES of data before we start.
+      @es.indices.delete_template(:name => "*")
+      # This can fail if there are no indexes, ignore failure.
+      @es.indices.delete(:index => "*") rescue nil
+      @es.index(
+        :index => 'logstash-update',
+        :type => doc_type,
+        :id => "123",
+        :body => { :message => 'Test', :counter => 1 }
+      )
+      @es.indices.refresh
+    end
+
+    it "should fail without a document_id" do
+      subject = get_es_output
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    context "when update only" do
+      it "should not create new document" do
+        subject = get_es_output({ 'document_id' => "456" } )
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+
+      it "should update existing document" do
+        subject = get_es_output({ 'document_id' => "123" })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
+        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+        expect(r["_source"]["message"]).to eq('updated message here')
+      end
+
+      # The es ruby client treats the data field differently. Make sure this doesn't
+      # raise an exception
+      it "should update an existing document that has a 'data' field" do
+        subject = get_es_output({ 'document_id' => "123" })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
+        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+        expect(r["_source"]["data"]).to eq('updated message here')
+        expect(r["_source"]["message"]).to eq('foo')
+      end
+
+      it "should allow default (internal) version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+        subject.register
+      end
+
+      it "should allow internal version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+        subject.register
+      end
+
+      it "should not allow external version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+
+      it "should not allow external_gt version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+
+      it "should not allow external_gte version" do
+        subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+
+    end
+
+    context "when update with upsert" do
+      it "should create new documents with provided upsert" do
+        subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+        expect(r["_source"]["message"]).to eq('upsert message')
+      end
+
+      it "should create new documents with event/doc as upsert" do
+        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+        subject.register
+        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+        r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+        expect(r["_source"]["message"]).to eq('sample message here')
+      end
+
+      it "should fail on documents with event/doc as upsert at external version" do
+        subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+        expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb

@@ -0,0 +1,19 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Update the password of the specified user
+      def delete_ilm_policy(arguments={})
+        method = HTTP_DELETE
+        path   = Utils.__pathify '_ilm/policy/',
+                                 Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/get_alias.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Retrieve the list of index lifecycle management policies
+      def get_alias(arguments={})
+        method = HTTP_GET
+        path   = Utils.__pathify '_alias', Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Retrieve the list of index lifecycle management policies
+      def get_ilm_policy(arguments={})
+        method = HTTP_GET
+        path   = Utils.__pathify '_ilm/policy', Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/put_alias.rb

@@ -0,0 +1,24 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # @option arguments [String] :name The name of the alias (*Required*)
+      # @option arguments [Hash]   :The alias definition(*Required*)
+
+      def put_alias(arguments={})
+        raise ArgumentError, "Required argument 'name' missing" unless arguments[:name]
+        raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+        method = HTTP_PUT
+        path   = Utils.__pathify Utils.__escape(arguments[:name])
+
+        params = Utils.__validate_and_extract_params arguments
+        body   = arguments[:body]
+        perform_request(method, path, params, body.to_json).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb

@@ -0,0 +1,25 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # @option arguments [String] :name The name of the policy (*Required*)
+      # @option arguments [Hash] :body The policy definition (*Required*)
+
+      def put_ilm_policy(arguments={})
+        raise ArgumentError, "Required argument 'name' missing" unless arguments[:name]
+        raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+        method = HTTP_PUT
+        path   = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name])
+
+        params = Utils.__validate_and_extract_params arguments
+
+        body   = arguments[:body]
+        perform_request(method, path, params, body.to_json).body
+      end
+    end
+  end
+end

data/spec/unit/http_client_builder_spec.rb

@@ -0,0 +1,185 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch"
+require "logstash/outputs/elasticsearch/http_client"
+require "logstash/outputs/elasticsearch/http_client_builder"
+
+describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
+  describe "auth setup with url encodable passwords" do
+    let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
+    let(:user) { "foo@bar"}
+    let(:password) {"baz@blah" }
+    let(:password_secured) do
+      secured = double("password")
+      allow(secured).to receive(:value).and_return(password)
+      secured
+    end
+    let(:options) { {"user" => user, "password" => password} }
+    let(:logger) { mock("logger") }
+    let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
+
+    it "should return the user escaped" do
+      expect(auth_setup[:user]).to eql(CGI.escape(user))
+    end
+
+    it "should return the password escaped" do
+      expect(auth_setup[:password]).to eql(CGI.escape(password))
+    end
+  end
+
+  describe "customizing action paths" do
+    let(:hosts) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] }
+    let(:options) { {"hosts" => hosts } }
+    let(:logger) { double("logger") }
+    before :each do
+      [:debug, :debug?, :info?, :info, :warn].each do |level|
+        allow(logger).to receive(level)
+      end
+    end
+
+    describe "healthcheck_path" do
+
+      context "when setting bulk_path" do
+        let(:bulk_path) { "/meh" }
+        let(:options) { super.merge("bulk_path" => bulk_path) }
+
+        context "when using path" do
+          let(:options) { super.merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the bulk_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting bulk_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super.merge("path" => path) }
+          it "sets bulk_path to path+_bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("#{path}/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the bulk_path to _bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "healthcheck_path" do
+      context "when setting healthcheck_path" do
+        let(:healthcheck_path) { "/meh" }
+        let(:options) { super.merge("healthcheck_path" => healthcheck_path) }
+
+        context "when using path" do
+          let(:options) { super.merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the healthcheck_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting healthcheck_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super.merge("path" => path) }
+          it "sets healthcheck_path to path" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the healthcheck_path to root" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq("/")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "sniffing_path" do
+      context "when setting sniffing_path" do
+        let(:sniffing_path) { "/meh" }
+        let(:options) { super.merge("sniffing_path" => sniffing_path) }
+
+        context "when using path" do
+          let(:options) { super.merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the sniffing_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting sniffing_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super.merge("path" => path) }
+          it "sets sniffing_path to path+_nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the sniffing_path to _nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb

@@ -0,0 +1,149 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/http_client"
+
+describe LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter do
+  let(:logger) { Cabin::Channel.get }
+  let(:options) { {} }
+
+  subject { described_class.new(logger, options) }
+
+  it "should raise an exception if requests are issued after close" do
+    subject.close
+    expect { subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, '/') }.to raise_error(::Manticore::ClientStoppedException)
+  end
+
+  it "should implement host unreachable exceptions" do
+    expect(subject.host_unreachable_exceptions).to be_a(Array)
+  end
+  
+  describe "auth" do
+    let(:user) { "myuser" }
+    let(:password) { "mypassword" }
+    let(:noauth_uri) { clone = uri.clone; clone.user=nil; clone.password=nil; clone }
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://#{user}:#{password}@localhost:9200") }
+    
+    it "should convert the auth to params" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(200)
+      
+      expected_uri = noauth_uri.clone
+      expected_uri.path = "/"
+      
+      expect(subject.manticore).to receive(:get).
+        with(expected_uri.to_s, {
+          :headers => {"Content-Type" => "application/json"},
+          :auth => {
+            :user => user,
+            :password => password,
+            :eager => true
+          }
+        }).and_return resp
+      
+      subject.perform_request(uri, :get, "/")
+    end
+  end
+
+  describe "bad response codes" do
+    let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+
+    it "should raise a bad response code error" do
+      resp = double("response")
+      allow(resp).to receive(:call)
+      allow(resp).to receive(:code).and_return(500)
+      allow(resp).to receive(:body).and_return("a body")
+
+      expect(subject.manticore).to receive(:get).
+        with(uri.to_s + "/", anything).
+        and_return(resp)
+
+      uri_with_path = uri.clone
+      uri_with_path.path = "/"
+
+      expect(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError).to receive(:new).
+        with(resp.code, uri_with_path, nil, resp.body).and_call_original
+
+      expect do
+        subject.perform_request(uri, :get, "/")
+      end.to raise_error(::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError)
+    end
+  end
+
+  describe "format_url" do
+    let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/") }
+    let(:path) { "_bulk" }
+    subject { described_class.new(double("logger"), {}) }
+
+    it "should add the path argument to the uri's path" do
+      expect(subject.format_url(url, path).path).to eq("/path/_bulk")
+    end
+
+    context "when uri contains query parameters" do
+      let(:query_params) { "query=value&key=value2" }
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200/path/?#{query_params}") }
+      let(:formatted) { subject.format_url(url, path)}
+
+      it "should retain query_params after format" do
+        expect(formatted.query).to eq(query_params)
+      end
+      
+      context "and the path contains query parameters" do
+        let(:path) { "/special_path?specialParam=123" }
+        
+        it "should join the query correctly" do
+          expect(formatted.query).to eq(query_params + "&specialParam=123")
+        end
+      end
+    end
+    
+    context "when the path contains query parameters" do
+      let(:path) { "/special_bulk?pathParam=1"}
+      let(:formatted) { subject.format_url(url, path) }
+      
+      it "should add the path correctly" do
+        expect(formatted.path).to eq("#{url.path}special_bulk")
+        expect(subject.remove_double_escaping(formatted.path)).to eq("#{url.path}special_bulk")
+      end 
+      
+      it "should add the query parameters correctly" do
+        expect(formatted.query).to eq("pathParam=1")
+      end
+    end
+
+    context "when uri contains credentials" do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://myuser:mypass@localhost:9200") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it "should remove credentials after format" do
+        expect(formatted.userinfo).to be_nil
+      end
+    end
+
+    context 'when uri contains date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("<logstash-{now/d}-0001>") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/%3Clogstash-%7Bnow%2Fd%7D-0001%3E")
+      end
+    end
+
+    context 'when uri does not contain date math' do
+      let(:url) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
+      let(:path) { CGI.escape("logstash-0001") }
+      let(:formatted) { subject.format_url(url, path) }
+
+      it 'should escape the uri correctly' do
+        expect(subject.remove_double_escaping(formatted.path)).to eq("/logstash-0001")
+      end
+    end
+  end
+
+  describe "integration specs", :integration => true do
+    it "should perform correct tests without error" do
+      resp = subject.perform_request(::LogStash::Util::SafeURI.new("http://localhost:9200"), :get, "/")
+      expect(resp.code).to eql(200)
+    end
+  end
+end