logstash-output-elasticsearch-test 11.16.0-x86_64-linux

data/spec/integration/outputs/templates_spec.rb

@@ -0,0 +1,133 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "index template expected behavior", :integration => true do
+  let(:ecs_compatibility) { fail('spec group does not define `ecs_compatibility`!') }
+
+  subject! do
+    require "logstash/outputs/elasticsearch"
+    allow_any_instance_of(LogStash::Outputs::ElasticSearch).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+
+    settings = {
+      "manage_template" => true,
+      "template_overwrite" => true,
+      "hosts" => "#{get_host_port()}"
+    }
+    next LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  let(:elasticsearch_client) { get_client }
+
+  before(:each) do
+    # delete indices and templates
+    require "elasticsearch"
+
+    elasticsearch_client.indices.delete_template(:name => '*')
+    # This can fail if there are no indexes, ignore failure.
+    elasticsearch_client.indices.delete(:index => '*') rescue puts("DELETE INDICES ERROR: #{$!}")
+    # Since we are pinned to ES client 7.x, we need to delete data streams the hard way...
+    elasticsearch_client.perform_request("DELETE", "/_data_stream/*") rescue puts("DELETE DATA STREAMS ERROR: #{$!}")
+  end
+
+  context 'with ecs_compatibility => disabled' do
+    let(:ecs_compatibility) { :disabled }
+    before :each do
+      @es = elasticsearch_client # cache as ivar for tests...
+
+      subject.register
+
+      subject.multi_receive([
+        LogStash::Event.new("message" => "sample message here"),
+        LogStash::Event.new("somemessage" => { "message" => "sample nested message here" }),
+        LogStash::Event.new("somevalue" => 100),
+        LogStash::Event.new("somevalue" => 10),
+        LogStash::Event.new("somevalue" => 1),
+        LogStash::Event.new("country" => "us"),
+        LogStash::Event.new("country" => "at"),
+        LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0 ] })
+      ])
+
+      @es.indices.refresh
+
+      # Wait or fail until everything's indexed.
+      Stud::try(20.times) do
+        r = @es.search(index: 'logstash*')
+        expect(r).to have_hits(8)
+      end
+    end
+
+    it "permits phrase searching on string fields" do
+      results = @es.search(index: 'logstash*', q: "message:\"sample message\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["message"]).to eq("sample message here")
+    end
+
+    it "numbers dynamically map to a numeric type and permit range queries" do
+      results = @es.search(index: 'logstash*', q: "somevalue:[5 TO 105]")
+      expect(results).to have_hits(2)
+
+      values = results["hits"]["hits"].collect { |r| r["_source"]["somevalue"] }
+      expect(values).to include(10)
+      expect(values).to include(100)
+      expect(values).to_not include(1)
+    end
+
+    it "does not create .keyword field for top-level message field" do
+      results = @es.search(index: 'logstash*', q: "message.keyword:\"sample message here\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "creates .keyword field for nested message fields" do
+      results = @es.search(index: 'logstash*', q: "somemessage.message.keyword:\"sample nested message here\"")
+      expect(results).to have_hits(1)
+    end
+
+    it "creates .keyword field from any string field which is not_analyzed" do
+      results = @es.search(index: 'logstash*', q: "country.keyword:\"us\"")
+      expect(results).to have_hits(1)
+      expect(results["hits"]["hits"][0]["_source"]["country"]).to eq("us")
+
+      # partial or terms should not work.
+      results = @es.search(index: 'logstash*', q: "country.keyword:\"u\"")
+      expect(results).to have_hits(0)
+    end
+
+    it "make [geoip][location] a geo_point" do
+      expect(field_properties_from_template("logstash", "geoip")["location"]["type"]).to eq("geo_point")
+    end
+
+    it "aggregate .keyword results correctly " do
+      results = @es.search(index: 'logstash*', body: { "aggregations" => { "my_agg" => { "terms" => { "field" => "country.keyword" } } } })["aggregations"]["my_agg"]
+      terms = results["buckets"].collect { |b| b["key"] }
+
+      expect(terms).to include("us")
+
+      # 'at' is a stopword, make sure stopwords are not ignored.
+      expect(terms).to include("at")
+    end
+  end
+
+  context 'with ECS enabled' do
+    let(:ecs_compatibility) { :v1 }
+
+    before(:each) do
+      subject.register # should load template?
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+    end
+
+    let(:elasticsearch_cluster_major_version) do
+      elasticsearch_client.info&.dig("version", "number" )&.split('.')&.map(&:to_i)&.first
+    end
+
+    it 'loads the templates' do
+      aggregate_failures do
+        if elasticsearch_cluster_major_version >= 8
+          # In ES 8+ we use the _index_template API
+          expect(elasticsearch_client.indices.exists_index_template(name: 'ecs-logstash')).to be_truthy
+        else
+          # Otherwise, we used the legacy _template API
+          expect(elasticsearch_client.indices.exists_template(name: 'ecs-logstash')).to be_truthy
+        end
+      end
+    end
+  end
+end

data/spec/integration/outputs/unsupported_actions_spec.rb

@@ -0,0 +1,75 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Unsupported actions testing...", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  INDEX = "logstash-unsupported-actions-rejected"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => INDEX,
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "%{action_field}",
+      "document_id" => "%{doc_id}",
+      "ecs_compatibility" => "disabled"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    # index single doc for update purpose
+    @es.index(
+      :index => INDEX,
+      :type => doc_type,
+      :id => "2",
+      :body => { :message => 'Test to doc indexing', :counter => 1 }
+    )
+    @es.index(
+      :index => INDEX,
+      :type => doc_type,
+      :id => "3",
+      :body => { :message => 'Test to doc deletion', :counter => 2 }
+    )
+    @es.indices.refresh
+  end
+
+  context "multiple actions include unsupported action" do
+    let(:events) {[
+      LogStash::Event.new("action_field" => "index", "doc_id" => 1, "message"=> "hello"),
+      LogStash::Event.new("action_field" => "update", "doc_id" => 2, "message"=> "hi"),
+      LogStash::Event.new("action_field" => "delete", "doc_id" => 3),
+      LogStash::Event.new("action_field" => "unsupported_action", "doc_id" => 4, "message"=> "world!")
+    ]}
+
+    it "should reject unsupported doc" do
+      subject = get_es_output
+      subject.register
+      subject.multi_receive(events)
+
+      index_or_update = proc do |event|
+        action = event.get("action_field")
+        action.eql?("index") || action.eql?("update")
+      end
+
+      indexed_events = events.select { |event| index_or_update.call(event) }
+      rejected_events = events.select { |event| !index_or_update.call(event) }
+
+      indexed_events.each do |event|
+        response = @es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)
+        expect(response['_source']['message']).to eq(event.get("message"))
+      end
+
+      rejected_events.each do |event|
+        expect {@es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+    end
+  end
+end

data/spec/integration/outputs/update_spec.rb

@@ -0,0 +1,114 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Update actions without scripts", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => doc_type,
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @es.indices.refresh
+  end
+
+  it "should fail without a document_id" do
+    subject = get_es_output
+    expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+  end
+
+  context "when update only" do
+    it "should not create new document" do
+      subject = get_es_output({ 'document_id' => "456" } )
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      expect {@es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+
+    it "should update existing document" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "updated message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["message"]).to eq('updated message here')
+    end
+
+    # The es ruby client treats the data field differently. Make sure this doesn't
+    # raise an exception
+    it "should update an existing document that has a 'data' field" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("data" => "updated message here", "message" => "foo")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "123", :refresh => true)
+      expect(r["_source"]["data"]).to eq('updated message here')
+      expect(r["_source"]["message"]).to eq('foo')
+    end
+
+    it "should allow default (internal) version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99" })
+      subject.register
+    end
+
+    it "should allow internal version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "internal" })
+      subject.register
+    end
+
+    it "should not allow external version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gt version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gt" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+    it "should not allow external_gte version" do
+      subject = get_es_output({ 'document_id' => "123", "version" => "99", "version_type" => "external_gte" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+
+  end
+
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('upsert message')
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => doc_type, :id => "456", :refresh => true)
+      expect(r["_source"]["message"]).to eq('sample message here')
+    end
+
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require "logstash/devutils/rspec/spec_helper"
+
+require "logstash/outputs/elasticsearch"
+
+module LogStash::Outputs::ElasticSearch::SpecHelper
+end
+
+RSpec.configure do |config|
+  config.include LogStash::Outputs::ElasticSearch::SpecHelper
+end

data/spec/support/elasticsearch/api/actions/delete_ilm_policy.rb

@@ -0,0 +1,19 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Update the password of the specified user
+      def delete_ilm_policy(arguments={})
+        method = HTTP_DELETE
+        path   = Utils.__pathify '_ilm/policy/',
+                                 Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/get_alias.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Retrieve the list of index lifecycle management policies
+      def get_alias(arguments={})
+        method = HTTP_GET
+        path   = Utils.__pathify '_alias', Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/get_ilm_policy.rb

@@ -0,0 +1,18 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # Retrieve the list of index lifecycle management policies
+      def get_ilm_policy(arguments={})
+        method = HTTP_GET
+        path   = Utils.__pathify '_ilm/policy', Utils.__escape(arguments[:name])
+        params = {}
+        perform_request(method, path, params, nil).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/put_alias.rb

@@ -0,0 +1,24 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # @option arguments [String] :name The name of the alias (*Required*)
+      # @option arguments [Hash]   :The alias definition(*Required*)
+
+      def put_alias(arguments={})
+        raise ArgumentError, "Required argument 'name' missing" unless arguments[:name]
+        raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+        method = HTTP_PUT
+        path   = Utils.__pathify Utils.__escape(arguments[:name])
+
+        params = Utils.__validate_and_extract_params arguments
+        body   = arguments[:body]
+        perform_request(method, path, params, body.to_json).body
+      end
+    end
+  end
+end

data/spec/support/elasticsearch/api/actions/put_ilm_policy.rb

@@ -0,0 +1,25 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+
+module Elasticsearch
+  module API
+    module Actions
+
+      # @option arguments [String] :name The name of the policy (*Required*)
+      # @option arguments [Hash] :body The policy definition (*Required*)
+
+      def put_ilm_policy(arguments={})
+        raise ArgumentError, "Required argument 'name' missing" unless arguments[:name]
+        raise ArgumentError, "Required argument 'body' missing" unless arguments[:body]
+        method = HTTP_PUT
+        path   = Utils.__pathify '_ilm/policy/', Utils.__escape(arguments[:name])
+
+        params = Utils.__validate_and_extract_params arguments
+
+        body   = arguments[:body]
+        perform_request(method, path, params, body.to_json).body
+      end
+    end
+  end
+end

data/spec/unit/http_client_builder_spec.rb

@@ -0,0 +1,185 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch"
+require "logstash/outputs/elasticsearch/http_client"
+require "logstash/outputs/elasticsearch/http_client_builder"
+
+describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
+  describe "auth setup with url encodable passwords" do
+    let(:klass) { LogStash::Outputs::ElasticSearch::HttpClientBuilder }
+    let(:user) { "foo@bar"}
+    let(:password) {"baz@blah" }
+    let(:password_secured) do
+      secured = double("password")
+      allow(secured).to receive(:value).and_return(password)
+      secured
+    end
+    let(:options) { {"user" => user, "password" => password} }
+    let(:logger) { mock("logger") }
+    let(:auth_setup) { klass.setup_basic_auth(double("logger"), {"user" => user, "password" => password_secured}) }
+
+    it "should return the user escaped" do
+      expect(auth_setup[:user]).to eql(CGI.escape(user))
+    end
+
+    it "should return the password escaped" do
+      expect(auth_setup[:password]).to eql(CGI.escape(password))
+    end
+  end
+
+  describe "customizing action paths" do
+    let(:hosts) { [ ::LogStash::Util::SafeURI.new("http://localhost:9200") ] }
+    let(:options) { {"hosts" => hosts } }
+    let(:logger) { double("logger") }
+    before :each do
+      [:debug, :debug?, :info?, :info, :warn].each do |level|
+        allow(logger).to receive(level)
+      end
+    end
+
+    describe "healthcheck_path" do
+
+      context "when setting bulk_path" do
+        let(:bulk_path) { "/meh" }
+        let(:options) { super().merge("bulk_path" => bulk_path) }
+
+        context "when using path" do
+          let(:options) { super().merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the bulk_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq(bulk_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting bulk_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super().merge("path" => path) }
+          it "sets bulk_path to path+_bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("#{path}/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the bulk_path to _bulk" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:bulk_path]).to eq("/_bulk")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "healthcheck_path" do
+      context "when setting healthcheck_path" do
+        let(:healthcheck_path) { "/meh" }
+        let(:options) { super().merge("healthcheck_path" => healthcheck_path) }
+
+        context "when using path" do
+          let(:options) { super().merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the healthcheck_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(healthcheck_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting healthcheck_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super().merge("path" => path) }
+          it "sets healthcheck_path to path" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq(path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the healthcheck_path to root" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:healthcheck_path]).to eq("/")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+    describe "sniffing_path" do
+      context "when setting sniffing_path" do
+        let(:sniffing_path) { "/meh" }
+        let(:options) { super().merge("sniffing_path" => sniffing_path) }
+
+        context "when using path" do
+          let(:options) { super().merge("path" => "/path") }
+          it "ignores the path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+        context "when not using path" do
+
+          it "uses the sniffing_path setting" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq(sniffing_path)
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+
+      context "when not setting sniffing_path" do
+
+        context "when using path" do
+          let(:path) { "/meh" }
+          let(:options) { super().merge("path" => path) }
+          it "sets sniffing_path to path+_nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+
+        context "when not using path" do
+          it "sets the sniffing_path to _nodes/http" do
+            expect(described_class).to receive(:create_http_client) do |options|
+              expect(options[:sniffing_path]).to eq("/_nodes/http")
+            end
+            described_class.build(logger, hosts, options)
+          end
+        end
+      end
+    end
+  end
+end